Example #1
0
        public void Read <T>(FieldCache[] fields, T entry, BitReader r, int recordOffset, Dictionary <long, string> stringsTable, FieldMetaData[] fieldMeta, ColumnMetaData[] columnMeta, Value32[][] palletData, Dictionary <int, Value32>[] commonData, int id, int refId, bool isSparse = false) where T : ClientDBRow
        {
            int fieldIndex = 0;

            foreach (var f in fields)
            {
                if (f.IsIndex && id != -1)
                {
                    ((FieldCache <T, int>)f).Setter(entry, id);
                    continue;
                }

                if (fieldIndex >= fieldMeta.Length)
                {
                    if (refId != -1)
                    {
                        ((FieldCache <T, int>)f).Setter(entry, refId);
                    }
                    continue;
                }

                if (f.IsArray)
                {
                    switch (f)
                    {
                    case FieldCache <T, int[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <int>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, uint[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <uint>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, byte[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <byte>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, sbyte[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <sbyte>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, short[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <short>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, ushort[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <ushort>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, float[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <float>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, long[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <long>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, ulong[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueArray <ulong>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
                        break;

                    case FieldCache <T, string[]> c1:
                        c1.Setter(entry, FieldReader.GetFieldValueStringsArray(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], stringsTable, isSparse, recordOffset, c1.ArraySize));
                        break;

                    default:
                        throw new Exception($"Unhandled DbcTable type: {f.Field.FieldType.FullName} in {f.Field.DeclaringType.FullName}.{f.Field.Name}");
                    }
                }
                else
                {
                    switch (f)
                    {
                    case FieldCache <T, int> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <int>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, uint> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <uint>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, byte> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <byte>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, sbyte> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <sbyte>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, short> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <short>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, ushort> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <ushort>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, float> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <float>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, long> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <long>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, ulong> c1:
                        c1.Setter(entry, FieldReader.GetFieldValue <ulong>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
                        break;

                    case FieldCache <T, string> c1:
                        c1.Setter(entry, isSparse ? r.ReadCString() : stringsTable[(recordOffset + (r.Position >> 3)) + FieldReader.GetFieldValue <int>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex])]);
                        break;

                    default:
                        throw new Exception($"Unhandled DbcTable type: {f.Field.FieldType.FullName} in {f.Field.DeclaringType.FullName}.{f.Field.Name}");
                    }
                }

                fieldIndex++;
            }
        }
Example #2
0
        public WDC3Reader(Stream stream, Func <ulong, bool> hasTactKey = null)
        {
            if (hasTactKey == null)
            {
                hasTactKeyFunc = (key) => false;
            }
            else
            {
                hasTactKeyFunc = hasTactKey;
            }

            using (var reader = new BinaryReader(stream, Encoding.UTF8))
            {
                if (reader.BaseStream.Length < HeaderSize)
                {
                    throw new InvalidDataException(String.Format("WDC3 file is corrupted!"));
                }

                uint magic = reader.ReadUInt32();

                if (magic != WDC3FmtSig)
                {
                    throw new InvalidDataException(String.Format("WDC3 file is corrupted!"));
                }

                RecordsCount    = reader.ReadInt32();
                FieldsCount     = reader.ReadInt32();
                RecordSize      = reader.ReadInt32();
                StringTableSize = reader.ReadInt32();
                TableHash       = reader.ReadUInt32();
                LayoutHash      = reader.ReadUInt32();
                MinIndex        = reader.ReadInt32();
                MaxIndex        = reader.ReadInt32();
                int locale = reader.ReadInt32();
                int flags  = reader.ReadUInt16();
                IdFieldIndex = reader.ReadUInt16();
                int totalFieldsCount   = reader.ReadInt32();
                int packedDataOffset   = reader.ReadInt32(); // Offset within the field where packed data starts
                int lookupColumnCount  = reader.ReadInt32(); // count of lookup columns
                int columnMetaDataSize = reader.ReadInt32(); // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
                int commonDataSize     = reader.ReadInt32();
                int palletDataSize     = reader.ReadInt32(); // in bytes, sizeof(DBC2PalletValue) == 4
                int sectionsCount      = reader.ReadInt32();

                //if (sectionsCount > 1)
                //    throw new Exception("sectionsCount > 1");

                SectionHeader_WDC3[] sections = reader.ReadArray <SectionHeader_WDC3>(sectionsCount);

                // field meta data
                m_meta = reader.ReadArray <FieldMetaData>(FieldsCount);

                if (sectionsCount == 0 || RecordsCount == 0)
                {
                    return;
                }

                // column meta data
                m_columnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);

                // pallet data
                m_palletData = new Value32[m_columnMeta.Length][];

                for (int i = 0; i < m_columnMeta.Length; i++)
                {
                    if (m_columnMeta[i].CompressionType == CompressionType.Pallet || m_columnMeta[i].CompressionType == CompressionType.PalletArray)
                    {
                        m_palletData[i] = reader.ReadArray <Value32>((int)m_columnMeta[i].AdditionalDataSize / 4);
                    }
                }

                // common data
                m_commonData = new Dictionary <int, Value32> [m_columnMeta.Length];

                for (int i = 0; i < m_columnMeta.Length; i++)
                {
                    if (m_columnMeta[i].CompressionType == CompressionType.Common)
                    {
                        Dictionary <int, Value32> commonValues = new Dictionary <int, Value32>();
                        m_commonData[i] = commonValues;

                        for (int j = 0; j < m_columnMeta[i].AdditionalDataSize / 8; j++)
                        {
                            commonValues[reader.ReadInt32()] = reader.Read <Value32>();
                        }
                    }
                }

                Dictionary <long, string> stringsTable = new Dictionary <long, string>();

                bool isSparse = (flags & 0x1) != 0;
                bool hasIndex = (flags & 0x4) != 0;

                int previousStringTableSize = 0, previousRecordCount = 0;

                for (int sectionIndex = 0; sectionIndex < sectionsCount; sectionIndex++)
                {
                    if (sections[sectionIndex].TactKeyLookup != 0 && !hasTactKeyFunc(sections[sectionIndex].TactKeyLookup))
                    {
                        //Console.WriteLine("Detected db2 with encrypted section! HasKey {0}", CASC.HasKey(Sections[sectionIndex].TactKeyLookup));
                        previousStringTableSize += sections[sectionIndex].StringTableSize;
                        previousRecordCount     += sections[sectionIndex].NumRecords;
                        continue;
                    }

                    reader.BaseStream.Position = sections[sectionIndex].FileOffset;

                    byte[]        recordsData;
                    SparseEntry[] sparseEntries = null;

                    if (isSparse)
                    {
                        // sparse data with inlined strings
                        recordsData = reader.ReadBytes(sections[sectionIndex].SparseDataEndOffset - sections[sectionIndex].FileOffset);

                        if (reader.BaseStream.Position != sections[sectionIndex].SparseDataEndOffset)
                        {
                            throw new Exception("reader.BaseStream.Position != sections[sectionIndex].SparseDataEndOffset");
                        }
                    }
                    else
                    {
                        // records data
                        recordsData = reader.ReadBytes(sections[sectionIndex].NumRecords * RecordSize);

                        // string data
                        for (int i = 0; i < sections[sectionIndex].StringTableSize;)
                        {
                            long oldPos = reader.BaseStream.Position;

                            stringsTable[i + previousStringTableSize] = CStringExtensions.ReadCString(reader);

                            i += (int)(reader.BaseStream.Position - oldPos);
                        }
                    }

                    Array.Resize(ref recordsData, recordsData.Length + 8); // pad with extra zeros so we don't crash when reading

                    // index data
                    int[] indexData = reader.ReadArray <int>(sections[sectionIndex].IndexDataSize / 4);

                    bool isIndexEmpty = hasIndex && indexData.Count(i => i == 0) == sections[sectionIndex].NumRecords;

                    // duplicate rows data
                    Dictionary <int, int> copyData = new Dictionary <int, int>();

                    for (int i = 0; i < sections[sectionIndex].NumCopyRecords; i++)
                    {
                        copyData[reader.ReadInt32()] = reader.ReadInt32();
                    }

                    if (sections[sectionIndex].NumSparseRecords > 0)
                    {
                        sparseEntries = reader.ReadArray <SparseEntry>(sections[sectionIndex].NumSparseRecords);
                    }

                    // reference data
                    ReferenceData refData = null;

                    if (sections[sectionIndex].ParentLookupDataSize > 0)
                    {
                        refData = new ReferenceData
                        {
                            NumRecords = reader.ReadInt32(),
                            MinId      = reader.ReadInt32(),
                            MaxId      = reader.ReadInt32()
                        };

                        ReferenceEntry[] entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);

                        refData.Entries = new Dictionary <int, int>();

                        for (int i = 0; i < entries.Length; i++)
                        {
                            refData.Entries[entries[i].Index] = entries[i].Id;
                        }
                    }
                    else
                    {
                        refData = new ReferenceData
                        {
                            Entries = new Dictionary <int, int>()
                        };
                    }

                    if (sections[sectionIndex].NumSparseRecords > 0)
                    {
                        // TODO: use this shit
                        int[] sparseIndexData = reader.ReadArray <int>(sections[sectionIndex].NumSparseRecords);

                        if (hasIndex && indexData.Length != sparseIndexData.Length)
                        {
                            throw new Exception("indexData.Length != sparseIndexData.Length");
                        }

                        indexData = sparseIndexData;
                    }

                    BitReader bitReader = new BitReader(recordsData);

                    if (sections[sectionIndex].NumSparseRecords > 0 && sections[sectionIndex].NumRecords != sections[sectionIndex].NumSparseRecords)
                    {
                        throw new Exception("sections[sectionIndex].NumSparseRecords > 0 && sections[sectionIndex].NumRecords != sections[sectionIndex].NumSparseRecords");
                    }

                    for (int i = 0; i < sections[sectionIndex].NumRecords; ++i)
                    {
                        bitReader.Position = 0;

                        if (isSparse)
                        {
                            bitReader.Offset = sparseEntries[i].Offset - sections[sectionIndex].FileOffset;
                        }
                        else
                        {
                            bitReader.Offset = i * RecordSize;
                        }

                        bool hasRef = refData.Entries.TryGetValue(i, out int refId);

                        int recordIndex  = i + previousRecordCount;
                        int recordOffset = (recordIndex * RecordSize) - (RecordsCount * RecordSize);

                        WDC3Row rec = new WDC3Row(this, bitReader, recordOffset, hasIndex ? (isIndexEmpty ? i : indexData[i]) : -1, hasRef ? refId : -1, isSparse, stringsTable);

                        _Records.Add(rec.GetId(), rec);

                        if (i % 1000 == 0)
                        {
                            Console.Write("\r{0} records read", i);
                        }
                    }

                    foreach (var copyRow in copyData)
                    {
                        WDC3Row rec = (WDC3Row)_Records[copyRow.Value].Clone();
                        rec.SetId(copyRow.Key);
                        _Records.Add(copyRow.Key, rec);
                    }

                    previousStringTableSize += sections[sectionIndex].StringTableSize;
                    previousRecordCount     += sections[sectionIndex].NumRecords;
                }
            }
        }
Example #3
0
        public WDC1Reader(Stream stream)
        {
            using (var reader = new BinaryReader(stream, Encoding.UTF8))
            {
                if (reader.BaseStream.Length < HeaderSize)
                {
                    throw new InvalidDataException(String.Format("WDC1 file is corrupted!"));
                }

                uint magic = reader.ReadUInt32();

                if (magic != WDC1FmtSig)
                {
                    throw new InvalidDataException(String.Format("WDC1 file is corrupted!"));
                }

                RecordsCount    = reader.ReadInt32();
                FieldsCount     = reader.ReadInt32();
                RecordSize      = reader.ReadInt32();
                StringTableSize = reader.ReadInt32();

                TableHash  = reader.ReadUInt32();
                LayoutHash = reader.ReadUInt32();
                MinIndex   = reader.ReadInt32();
                MaxIndex   = reader.ReadInt32();
                int locale        = reader.ReadInt32();
                int copyTableSize = reader.ReadInt32();
                int flags         = reader.ReadUInt16();
                IdFieldIndex = reader.ReadUInt16();

                int totalFieldsCount   = reader.ReadInt32();
                int packedDataOffset   = reader.ReadInt32(); // Offset within the field where packed data starts
                int lookupColumnCount  = reader.ReadInt32(); // count of lookup columns
                int sparseTableOffset  = reader.ReadInt32(); // absolute value, {uint offset, ushort size}[MaxId - MinId + 1]
                int indexDataSize      = reader.ReadInt32(); // int indexData[IndexDataSize / 4]
                int columnMetaDataSize = reader.ReadInt32(); // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
                int commonDataSize     = reader.ReadInt32();
                int palletDataSize     = reader.ReadInt32(); // in bytes, sizeof(DBC2PalletValue) == 4
                int referenceDataSize  = reader.ReadInt32(); // uint NumRecords, uint minId, uint maxId, {uint id, uint index}[NumRecords], questionable usefulness...

                // field meta data
                m_meta = reader.ReadArray <FieldMetaData>(FieldsCount);

                byte[] recordsData;
                Dictionary <long, string> stringsTable  = null;
                SparseEntry[]             sparseEntries = null;

                if ((flags & 0x1) == 0)
                {
                    // records data
                    recordsData = reader.ReadBytes(RecordsCount * RecordSize);

                    Array.Resize(ref recordsData, recordsData.Length + 8); // pad with extra zeros so we don't crash when reading

                    // string data
                    stringsTable = new Dictionary <long, string>();

                    for (int i = 0; i < StringTableSize;)
                    {
                        long oldPos = reader.BaseStream.Position;

                        stringsTable[i] = CStringExtensions.ReadCString(reader);

                        i += (int)(reader.BaseStream.Position - oldPos);
                    }
                }
                else
                {
                    // sparse data with inlined strings
                    recordsData = reader.ReadBytes(sparseTableOffset - HeaderSize - Marshal.SizeOf <FieldMetaData>() * FieldsCount);

                    if (reader.BaseStream.Position != sparseTableOffset)
                    {
                        throw new Exception("r.BaseStream.Position != sparseTableOffset");
                    }

                    sparseEntries = reader.ReadArray <SparseEntry>(MaxIndex - MinIndex + 1);

                    if (sparseTableOffset != 0)
                    {
                        throw new Exception("Sparse Table NYI!");
                    }
                    else
                    {
                        throw new Exception("Sparse Table with zero offset?");
                    }
                }

                // index data
                int[] indexData = reader.ReadArray <int>(indexDataSize / 4);

                // duplicate rows data
                Dictionary <int, int> copyData = new Dictionary <int, int>();

                for (int i = 0; i < copyTableSize / 8; i++)
                {
                    copyData[reader.ReadInt32()] = reader.ReadInt32();
                }

                // column meta data
                m_columnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);

                // pallet data
                m_palletData = new Value32[m_columnMeta.Length][];

                for (int i = 0; i < m_columnMeta.Length; i++)
                {
                    if (m_columnMeta[i].CompressionType == CompressionType.Pallet || m_columnMeta[i].CompressionType == CompressionType.PalletArray)
                    {
                        m_palletData[i] = reader.ReadArray <Value32>((int)m_columnMeta[i].AdditionalDataSize / 4);
                    }
                }

                // common data
                m_commonData = new Dictionary <int, Value32> [m_columnMeta.Length];

                for (int i = 0; i < m_columnMeta.Length; i++)
                {
                    if (m_columnMeta[i].CompressionType == CompressionType.Common)
                    {
                        Dictionary <int, Value32> commonValues = new Dictionary <int, Value32>();
                        m_commonData[i] = commonValues;

                        for (int j = 0; j < m_columnMeta[i].AdditionalDataSize / 8; j++)
                        {
                            commonValues[reader.ReadInt32()] = reader.Read <Value32>();
                        }
                    }
                }

                // reference data
                ReferenceData refData = null;

                if (referenceDataSize > 0)
                {
                    refData = new ReferenceData
                    {
                        NumRecords = reader.ReadInt32(),
                        MinId      = reader.ReadInt32(),
                        MaxId      = reader.ReadInt32()
                    };

                    ReferenceEntry[] entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);
                    refData.Entries = entries.ToDictionary(e => e.Index, e => e.Id);
                }
                else
                {
                    refData = new ReferenceData
                    {
                        Entries = new Dictionary <int, int>()
                    };
                }

                BitReader bitReader = new BitReader(recordsData);

                for (int i = 0; i < RecordsCount; ++i)
                {
                    bitReader.Position = 0;
                    bitReader.Offset   = i * RecordSize;

                    bool hasRef = refData.Entries.TryGetValue(i, out int refId);

                    WDC1Row rec = new WDC1Row(this, bitReader, indexDataSize != 0 ? indexData[i] : -1, hasRef ? refId : -1, stringsTable);

                    if (indexDataSize != 0)
                    {
                        _Records.Add(indexData[i], rec);
                    }
                    else
                    {
                        _Records.Add(rec.GetId(), rec);
                    }

                    if (i % 1000 == 0)
                    {
                        Console.Write("\r{0} records read", i);
                    }
                }

                foreach (var copyRow in copyData)
                {
                    WDC1Row rec = (WDC1Row)_Records[copyRow.Value].Clone();
                    rec.SetId(copyRow.Key);
                    _Records.Add(copyRow.Key, rec);
                }
            }
        }