Exemple #1
0
        private static uint GetHash(string str)
        {
            uint hash;

            if (stringHashLookup.ContainsKey(str))
            {
                hash = stringHashLookup[str];
            }
            else
            {
                hash = Checksum.FFnv32(str);
                stringHashLookup.Add(str, hash);
            }
            return(hash);
        }
Exemple #2
0
        public override void Read(BinaryStream bs)
        {
            Stopwatch sw = new Stopwatch();

            sw.Start();

            // read header
            HeaderInfo headerInfo = bs.Read.Type <HeaderInfo>();

            this.Patch       = headerInfo.patchName;
            this.Timestamp   = new DateTime();
            this.Timestamp   = Util.Time.DateTimeFromUnixTimestampMicroseconds((long)headerInfo.timestamp);
            this.Flags       = headerInfo.flags;
            this.fileVersion = headerInfo.version;

            //Console.WriteLine("read: "+sw.ElapsedMilliseconds);
            sw.Restart();

            // deobfuscate

            byte[] data = bs.Read.ByteArray((int)headerInfo.payloadSize);
            MTXor(Checksum.FFnv32(headerInfo.patchName), ref data);

            //Console.WriteLine("dxor: " + sw.ElapsedMilliseconds);
            sw.Restart();

            // cleanup memory, the original stream is not needed anymore
            bs.Dispose();
            bs = null;
            GC.Collect();

            // read compression header
            // uint inflated size
            // uint padding
            // ushort 0x78 0x01 zlib deflate low/no compression

            uint   inflatedSize = UIntFromBufferLE(ref data);
            ushort ds           = UShortFromBufferLE(ref data, 8);

            byte[] inflated = new byte[inflatedSize];
            Inflate(data, ref inflated, SharpCompress.Compressors.Deflate.CompressionLevel.BestSpeed, (int)inflatedSize, 10);
            BinaryStream ibs = new BinaryStream(new MemoryStream((inflated)));

            data = null;


            //Console.WriteLine("infl: " + sw.ElapsedMilliseconds);
            sw.Restart();

            // cleanup memory, the deobfuscated stream is not needed anymore
            GC.Collect();

            // read table header
            this.tableVersion = ibs.Read.UInt();
            ushort indexLength = ibs.Read.UShort();

            // read table info
            TableInfo[] tableInfos = new TableInfo[indexLength];
            for (ushort i = 0; i < indexLength; i++)
            {
                tableInfos[i] = ibs.Read.Type <TableInfo>();
            }

            // read field info
            FieldInfo[][] fieldInfos = new FieldInfo[indexLength][];
            for (int i = 0; i < indexLength; i++)
            {
                fieldInfos[i] = new FieldInfo[tableInfos[i].numFields];
                for (int x = 0; x < tableInfos[i].numFields; x++)
                {
                    fieldInfos[i][x] = ibs.Read.Type <FieldInfo>();
                }
            }

            // read row info
            RowInfo[] rowInfos = new RowInfo[indexLength];
            for (ushort i = 0; i < indexLength; i++)
            {
                rowInfos[i] = ibs.Read.Type <RowInfo>();
            }


            // build tables
            Tables = new List <Table>(indexLength);
            for (ushort i = 0; i < indexLength; i++)
            {
                TableInfo   tableInfo = tableInfos[i];
                FieldInfo[] fieldInfo = fieldInfos[i];
                RowInfo     rowInfo   = rowInfos[i];

                // setup table
                Table table = new Table();
                table.Id = tableInfo.id;

                // add fields
                table.Columns = new List <Column>(tableInfo.numFields);
                int currentWidth = 0;
                for (int x = 0; x < tableInfo.numFields; x++)
                {
                    Column field = new Column();
                    field.Id   = fieldInfos[i][x].id;
                    field.Type = (DBType)fieldInfos[i][x].type;

                    // fix removed fields? (weird padding some places)
                    if (fieldInfo[x].start != currentWidth)
                    {
                        int padding = fieldInfo[x].start - currentWidth;
                        field.Padding = padding;
                        currentWidth += padding;
                    }
                    currentWidth += DBTypeLength((DBType)fieldInfo[x].type);


                    table.Columns.Add(field);
                }

                // if any, add nullable fields
                if (tableInfo.nullableBitfields != 0)
                {
                    int count = 0;
                    for (int x = 0; x < tableInfo.numFields; x++)
                    {
                        if (fieldInfos[i][x].nullableIndex != 255)
                        {
                            count++;
                        }
                    }

                    Column[] nullableColumns = new Column[count];
                    for (int x = 0; x < tableInfo.numFields; x++)
                    {
                        if (fieldInfos[i][x].nullableIndex != 255)
                        {
                            nullableColumns[fieldInfos[i][x].nullableIndex] = table.Columns[x];
                        }
                    }
                    table.NullableColumn = new List <Column>(nullableColumns);
                }
                else
                {
                    table.NullableColumn = new List <Column>();
                }

                Tables.Add(table);
            }

            //Console.WriteLine("tabl: " + sw.ElapsedMilliseconds);
            sw.Restart();


            // read rows
            ConcurrentQueue <int> tableRowsReadQueue = new ConcurrentQueue <int>();

            for (ushort i = 0; i < indexLength; i++)
            {
                tableRowsReadQueue.Enqueue(i);
            }

            Parallel.For(0, numThreads, new ParallelOptions {
                MaxDegreeOfParallelism = numThreads
            }, q => {
                BinaryStream dbs = new BinaryStream(new MemoryStream(inflated));

                while (tableRowsReadQueue.Count != 0)
                {
                    int i;
                    if (!tableRowsReadQueue.TryDequeue(out i))
                    {
                        continue;
                    }

                    TableInfo tableInfo   = tableInfos[i];
                    FieldInfo[] fieldInfo = fieldInfos[i];
                    RowInfo rowInfo       = rowInfos[i];

                    Tables[i].Rows = new List <Row>();

                    for (int y = 0; y < rowInfo.rowCount; y++)
                    {
                        Row row        = new Row(tableInfo.numFields);
                        dbs.ByteOffset = rowInfo.rowOffset + (tableInfo.numBytes * y) + fieldInfo[0].start;
                        for (int z = 0; z < tableInfo.numFields; z++)
                        {
                            if (Tables[i].Columns[z].Padding != 0)
                            {
                                dbs.ByteOffset += Tables[i].Columns[z].Padding;
                            }
                            // just read the basic type now, unpack & decrypt later to reduce seeking
                            row.Fields.Add(ReadDBType(dbs, (DBType)fieldInfo[z].type));
                        }

                        // null out nulls again :P
                        if (tableInfo.nullableBitfields > 0)
                        {
                            byte[] nulls = dbs.Read.BitArray(tableInfo.nullableBitfields * 8);
                            for (int n = 0; n < Tables[i].NullableColumn.Count; n++)
                            {
                                if (nulls[n] == 1)
                                {
                                    int index  = Tables[i].Columns.IndexOf(Tables[i].NullableColumn[n]);
                                    row[index] = null;
                                }
                            }
                        }
                        Tables[i].Rows.Add(row);
                    }
                }
            });

            inflated = null;
            //Console.WriteLine("rows: " + sw.ElapsedMilliseconds);
            sw.Restart();

            // seek to the very end of the tables/start of data
            RowInfo   lri = rowInfos[rowInfos.Length - 1];
            TableInfo lti = tableInfos[tableInfos.Length - 1];

            ibs.ByteOffset = lri.rowOffset + (lri.rowCount * lti.numBytes);

            // copy the data to a new stream
            int dataLength = (int)(ibs.Length - ibs.ByteOffset);

            byte[] dataBlock = ibs.Read.ByteArray(dataLength);

            // cleanup
            ibs.Dispose();
            ibs = null;
            GC.Collect();

            // get unique data entry keys
            HashSet <uint>         uniqueKeys  = new HashSet <uint>();
            ConcurrentQueue <uint> uniqueQueue = new ConcurrentQueue <uint>();

            uniqueEntries = new Dictionary <uint, byte[]>();

            for (int i = 0; i < Tables.Count; i++)
            {
                for (int x = 0; x < Tables[i].Columns.Count; x++)
                {
                    DBType type = Tables[i].Columns[x].Type;

                    if (IsDataType(type))
                    {
                        for (int y = 0; y < Tables[i].Rows.Count; y++)
                        {
                            uint?k = (uint?)Tables[i].Rows[y][x];
                            if (k != null)
                            {
                                if (!uniqueKeys.Contains((uint)k))
                                {
                                    uniqueKeys.Add((uint)k);
                                    uniqueQueue.Enqueue((uint)k);
                                }
                            }
                        }
                    }
                }
            }

            //Console.WriteLine("uniq: " + sw.ElapsedMilliseconds);
            sw.Restart();

            // unpack & decrypt unique data entries to cache
            Parallel.For(0, numThreads, new ParallelOptions {
                MaxDegreeOfParallelism = numThreads
            }, i => {
                BinaryStream dbs = new BinaryStream(new MemoryStream(dataBlock));

                while (uniqueQueue.Count != 0)
                {
                    uint key;
                    if (!uniqueQueue.TryDequeue(out key))
                    {
                        continue;
                    }
                    byte[] d = GetDataEntry(dbs, key);

                    lock (uniqueEntries)
                    {
                        uniqueEntries.Add(key, d);
                    }
                }
                dbs.Dispose();
            });
            dataBlock = null;

            //Console.WriteLine("upac: " + sw.ElapsedMilliseconds);
            sw.Restart();

            // copy data entires to the tables from cache
            for (int z = 0; z < Tables.Count; z++)
            {
                for (int x = 0; x < Tables[z].Columns.Count; x++)
                {
                    DBType type = Tables[z].Columns[x].Type;
                    if (IsDataType(type))
                    {
                        Parallel.For(0, Tables[z].Rows.Count, y =>
                        {
                            uint?k     = (uint?)Tables[z].Rows[y][x];
                            object obj = null;
                            if (k != null)
                            {
                                if (uniqueEntries.ContainsKey((uint)k))
                                {
                                    byte[] d = uniqueEntries[(uint)k];
                                    if (d != null)
                                    {
                                        obj = BytesToDBType(type, d);
                                    }
                                }
                            }
                            Tables[z].Rows[y][x] = obj;
                        });
                    }
                }
            }


            //Console.WriteLine("assi: " + sw.ElapsedMilliseconds);
            sw.Restart();

            // cleanup :>
            uniqueKeys  = null;
            uniqueQueue = null;
            //uniqueEntries = null; // dont clean up these in case you need to look up data entries post load
            headerInfo = null;
            tableInfos = null;
            fieldInfos = null;
            rowInfos   = null;
            GC.Collect();
        }