public new Dictionary <int, byte[]> ReadOffsetData(BinaryReader dbReader, long pos) { Dictionary <int, byte[]> CopyTable = new Dictionary <int, byte[]>(); List <Tuple <int, short> > offsetmap = new List <Tuple <int, short> >(); Dictionary <int, OffsetDuplicate> firstindex = new Dictionary <int, OffsetDuplicate>(); long commonDataTablePos = dbReader.BaseStream.Length - CommonDataTableSize; long copyTablePos = commonDataTablePos - CopyTableSize; long indexTablePos = copyTablePos - (HasIndexTable ? RecordCount * 4 : 0); int[] m_indexes = null; //Offset Map if (HasOffsetTable) { // Records table dbReader.Scrub(StringBlockSize); for (int i = 0; i < (MaxId - MinId + 1); i++) { int offset = dbReader.ReadInt32(); short length = dbReader.ReadInt16(); if (offset == 0 || length == 0) { continue; } //Special case, may contain duplicates in the offset map that we don't want if (CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); } else { OffsetDuplicates.Add(MinId + i, firstindex[offset].VisibleIndex); } } offsetmap.Add(new Tuple <int, short>(offset, length)); } } //Index table if (HasIndexTable) { //Offset map alone reads straight into this others may not if (!HasOffsetTable || HasSecondIndex) { dbReader.Scrub(indexTablePos); } m_indexes = new int[RecordCount]; for (int i = 0; i < RecordCount; i++) { m_indexes[i] = dbReader.ReadInt32(); } } //Extract record data for (int i = 0; i < Math.Max(RecordCount, offsetmap.Count); i++) { if (HasOffsetTable) { int id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; if (CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) //Ignore duplicates { continue; } dbReader.Scrub(map.Item1); IEnumerable <byte> recordbytes = BitConverter.GetBytes(id) .Concat(dbReader.ReadBytes(map.Item2)); CopyTable.Add(id, recordbytes.ToArray()); } else { dbReader.Scrub(pos + i * RecordSize); byte[] recordbytes = dbReader.ReadBytes((int)RecordSize).ToArray(); if (HasIndexTable) { IEnumerable <byte> newrecordbytes = BitConverter.GetBytes(m_indexes[i]).Concat(recordbytes); CopyTable.Add(m_indexes[i], newrecordbytes.ToArray()); } else { int bytecount = FieldStructure[IdIndex].ByteCount; int offset = FieldStructure[IdIndex].Offset; int id = 0; for (int j = 0; j < bytecount; j++) { id |= (recordbytes[offset + j] << (j * 8)); } CopyTable.Add(id, recordbytes); } } } //CopyTable if (CopyTableSize != 0 && copyTablePos != dbReader.BaseStream.Length) { dbReader.Scrub(copyTablePos); while (dbReader.BaseStream.Position != dbReader.BaseStream.Length) { int id = dbReader.ReadInt32(); int idcopy = dbReader.ReadInt32(); byte[] copyRow = CopyTable[idcopy]; byte[] newRow = new byte[copyRow.Length]; Array.Copy(copyRow, newRow, newRow.Length); Array.Copy(BitConverter.GetBytes(id), newRow, sizeof(int)); CopyTable.Add(id, newRow); } } //CommonDataTable if (CommonDataTableSize > 0) { dbReader.Scrub(commonDataTablePos); int columncount = dbReader.ReadInt32(); var commondatalookup = new Dictionary <int, byte[]> [columncount]; //Initial Data extraction for (int i = 0; i < columncount; i++) { int count = dbReader.ReadInt32(); byte type = dbReader.ReadByte(); short bit = CommonDataBits[type]; int size = (32 - bit) >> 3; commondatalookup[i] = new Dictionary <int, byte[]>(); //New field not defined in header if (i > FieldStructure.Count - 1) { var offset = (ushort)((FieldStructure.Count == 0 ? 0 : FieldStructure[i - 1].Offset + FieldStructure[i - 1].ByteCount)); FieldStructure.Add(new FieldStructureEntry(bit, offset, type)); if (FieldStructure.Count > 1) { FieldStructure[i - 1].SetLength(FieldStructure[i]); } } for (int x = 0; x < count; x++) { commondatalookup[i].Add(dbReader.ReadInt32(), dbReader.ReadBytes(size)); } } var ids = CopyTable.Keys.ToArray(); foreach (var id in ids) { for (int i = 0; i < commondatalookup.Length; i++) { if (!FieldStructure[i].CommonDataColumn) { continue; } var col = commondatalookup[i]; var defaultValue = TableStructure?.Fields?[i]?.DefaultValue; defaultValue = string.IsNullOrEmpty(defaultValue) ? "0" : defaultValue; var field = FieldStructure[i]; var zeroData = new byte[field.ByteCount]; if (defaultValue != "0") { switch (field.CommonDataType) { case 1: zeroData = BitConverter.GetBytes(ushort.Parse(defaultValue)); break; case 2: zeroData = new[] { byte.Parse(defaultValue) }; break; case 3: zeroData = BitConverter.GetBytes(float.Parse(defaultValue)); break; case 4: zeroData = BitConverter.GetBytes(int.Parse(defaultValue)); break; } } byte[] currentData = CopyTable[id]; byte[] data = col.ContainsKey(id) ? col[id] : zeroData; Array.Resize(ref currentData, currentData.Length + data.Length); Array.Copy(data, 0, currentData, field.Offset, data.Length); CopyTable[id] = currentData; } } commondatalookup = null; RawRecordSize = (uint)CopyTable.Values.First().Length; } return(CopyTable); }
public new Dictionary <int, byte[]> ReadOffsetData(BinaryReader dbReader, long pos) { Dictionary <int, byte[]> CopyTable = new Dictionary <int, byte[]>(); List <Tuple <int, short> > offsetmap = new List <Tuple <int, short> >(); Dictionary <int, OffsetDuplicate> firstindex = new Dictionary <int, OffsetDuplicate>(); Dictionary <int, List <int> > Copies = new Dictionary <int, List <int> >(); columnOffsets = new List <int>(); recordOffsets = new List <int>(); int[] m_indexes = null; // OffsetTable if (HasOffsetTable && OffsetTableOffset > 0) { dbReader.BaseStream.Position = OffsetTableOffset; for (int i = 0; i < (MaxId - MinId + 1); i++) { int offset = dbReader.ReadInt32(); short length = dbReader.ReadInt16(); if (offset == 0 || length == 0) { continue; } // special case, may contain duplicates in the offset map that we don't want if (CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); } else { continue; } } offsetmap.Add(new Tuple <int, short>(offset, length)); } } // IndexTable if (HasIndexTable) { m_indexes = new int[RecordCount]; for (int i = 0; i < RecordCount; i++) { m_indexes[i] = dbReader.ReadInt32(); } } // Copytable if (CopyTableSize > 0) { long end = dbReader.BaseStream.Position + CopyTableSize; while (dbReader.BaseStream.Position < end) { int id = dbReader.ReadInt32(); int idcopy = dbReader.ReadInt32(); if (!Copies.ContainsKey(idcopy)) { Copies.Add(idcopy, new List <int>()); } Copies[idcopy].Add(id); } } // Relationships if (RelationshipDataSize > 0) { RelationShipData = new RelationShipData() { Records = dbReader.ReadUInt32(), MinId = dbReader.ReadUInt32(), MaxId = dbReader.ReadUInt32(), Entries = new Dictionary <uint, byte[]>() }; for (int i = 0; i < RelationShipData.Records; i++) { byte[] foreignKey = dbReader.ReadBytes(4); uint index = dbReader.ReadUInt32(); // has duplicates just like the copy table does... why? if (!RelationShipData.Entries.ContainsKey(index)) { RelationShipData.Entries.Add(index, foreignKey); } } FieldStructure.Add(new FieldStructureEntry(0, 0)); ColumnMeta.Add(new ColumnStructureEntry()); } // Record Data BitStream bitStream = new BitStream(recordData); for (int i = 0; i < RecordCount; i++) { int id = 0; if (HasOffsetTable && HasIndexTable) { id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; if (CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) //Ignore duplicates { continue; } dbReader.BaseStream.Position = map.Item1; byte[] data = dbReader.ReadBytes(map.Item2); IEnumerable <byte> recordbytes = BitConverter.GetBytes(id).Concat(data); // append relationship id if (RelationShipData != null) { // seen cases of missing indicies if (RelationShipData.Entries.TryGetValue((uint)i, out byte[] foreignData))
public new Dictionary <int, byte[]> ReadOffsetData(BinaryReader dbReader, long pos) { var CopyTable = new Dictionary <int, byte[]>(); var offsetmap = new List <Tuple <int, short> >(); var firstindex = new Dictionary <int, OffsetDuplicate>(); var OffsetDuplicates = new Dictionary <int, int>(); var Copies = new Dictionary <int, List <int> >(); int[] m_indexes = null; // OffsetTable if (HasOffsetTable && OffsetTableOffset > 0) { dbReader.BaseStream.Position = OffsetTableOffset; for (var i = 0; i < MaxId - MinId + 1; i++) { int offset = dbReader.ReadInt32(); short length = dbReader.ReadInt16(); if (offset == 0 || length == 0) { continue; } // special case, may contain duplicates in the offset map that we don't want if (CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); } else { OffsetDuplicates.Add(MinId + i, firstindex[offset].VisibleIndex); continue; } } offsetmap.Add(new Tuple <int, short>(offset, length)); } } // IndexTable if (HasIndexTable) { m_indexes = new int[RecordCount]; for (var i = 0; i < RecordCount; i++) { m_indexes[i] = dbReader.ReadInt32(); } } // Copytable if (CopyTableSize > 0) { long end = dbReader.BaseStream.Position + CopyTableSize; while (dbReader.BaseStream.Position < end) { int id = dbReader.ReadInt32(); int idcopy = dbReader.ReadInt32(); if (!Copies.ContainsKey(idcopy)) { Copies.Add(idcopy, new List <int>()); } Copies[idcopy].Add(id); } } // ColumnMeta ColumnMeta = new List <ColumnStructureEntry>(); for (var i = 0; i < FieldCount; i++) { ColumnStructureEntry column = new ColumnStructureEntry { RecordOffset = dbReader.ReadUInt16(), Size = dbReader.ReadUInt16(), AdditionalDataSize = dbReader.ReadUInt32(), // size of pallet / sparse values CompressionType = (CompressionType)dbReader.ReadUInt32(), BitOffset = dbReader.ReadInt32(), BitWidth = dbReader.ReadInt32(), Cardinality = dbReader.ReadInt32() }; // preload arraysizes if (column.CompressionType == CompressionType.None) { column.ArraySize = Math.Max(column.Size / FieldStructure[i].BitCount, 1); } else if (column.CompressionType == CompressionType.PalletArray) { column.ArraySize = Math.Max(column.Cardinality, 1); } ColumnMeta.Add(column); } // Pallet values for (var i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray) { int elements = (int)ColumnMeta[i].AdditionalDataSize / 4; int cardinality = Math.Max(ColumnMeta[i].Cardinality, 1); ColumnMeta[i].PalletValues = new List <byte[]>(); for (var j = 0; j < elements / cardinality; j++) { ColumnMeta[i].PalletValues.Add(dbReader.ReadBytes(cardinality * 4)); } } } // Sparse values for (var i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Sparse) { ColumnMeta[i].SparseValues = new Dictionary <int, byte[]>(); for (var j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++) { ColumnMeta[i].SparseValues[dbReader.ReadInt32()] = dbReader.ReadBytes(4); } } } // Relationships if (RelationshipDataSize > 0) { RelationShipData = new RelationShipData { Records = dbReader.ReadUInt32(), MinId = dbReader.ReadUInt32(), MaxId = dbReader.ReadUInt32(), Entries = new Dictionary <uint, byte[]>() }; for (var i = 0; i < RelationShipData.Records; i++) { byte[] foreignKey = dbReader.ReadBytes(4); uint index = dbReader.ReadUInt32(); // has duplicates just like the copy table does... why? if (!RelationShipData.Entries.ContainsKey(index)) { RelationShipData.Entries.Add(index, foreignKey); } } FieldStructure.Add(new FieldStructureEntry(0, 0)); ColumnMeta.Add(new ColumnStructureEntry()); } // Record Data BitStream bitStream = new(recordData); for (var i = 0; i < RecordCount; i++) { var id = 0; if (HasOffsetTable && HasIndexTable) { id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; if (CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) //Ignore duplicates { continue; } dbReader.BaseStream.Position = map.Item1; byte[] data = dbReader.ReadBytes(map.Item2); var recordbytes = BitConverter.GetBytes(id).Concat(data); // append relationship id if (RelationShipData != null) { // seen cases of missing indicies if (RelationShipData.Entries.TryGetValue((uint)i, out byte[] foreignData))
public new Dictionary <int, byte[]> ReadOffsetData(BinaryReader dbReader, long pos) { Dictionary <int, byte[]> CopyTable = new Dictionary <int, byte[]>(); List <Tuple <int, short> > offsetmap = new List <Tuple <int, short> >(); Dictionary <int, OffsetDuplicate> firstindex = new Dictionary <int, OffsetDuplicate>(); Dictionary <int, int> OffsetDuplicates = new Dictionary <int, int>(); Dictionary <int, List <int> > Copies = new Dictionary <int, List <int> >(); int[] m_indexes = null; // OffsetTable if (HasOffsetTable && OffsetTableOffset > 0) { dbReader.BaseStream.Position = OffsetTableOffset; for (int i = 0; i < (MaxId - MinId + 1); i++) { int offset = dbReader.ReadInt32(); short length = dbReader.ReadInt16(); if (offset == 0 || length == 0) { continue; } // special case, may contain duplicates in the offset map that we don't want if (CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); } else { OffsetDuplicates.Add(MinId + i, firstindex[offset].VisibleIndex); } } offsetmap.Add(new Tuple <int, short>(offset, length)); } } // IndexTable if (HasIndexTable) { m_indexes = new int[RecordCount]; for (int i = 0; i < RecordCount; i++) { m_indexes[i] = dbReader.ReadInt32(); } } // Copytable if (CopyTableSize > 0) { long end = dbReader.BaseStream.Position + CopyTableSize; while (dbReader.BaseStream.Position < end) { int id = dbReader.ReadInt32(); int idcopy = dbReader.ReadInt32(); if (!Copies.ContainsKey(idcopy)) { Copies.Add(idcopy, new List <int>()); } Copies[idcopy].Add(id); } } // ColumnMeta ColumnMeta = new List <ColumnStructureEntry>(); for (int i = 0; i < FieldCount; i++) { var column = new ColumnStructureEntry() { RecordOffset = dbReader.ReadUInt16(), Size = dbReader.ReadUInt16(), AdditionalDataSize = dbReader.ReadUInt32(), // size of pallet / sparse values CompressionType = (CompressionType)dbReader.ReadUInt32(), BitOffset = dbReader.ReadInt32(), BitWidth = dbReader.ReadInt32(), Cardinality = dbReader.ReadInt32() }; // preload arraysizes if (column.CompressionType == CompressionType.None) { column.ArraySize = Math.Max(column.Size / FieldStructure[i].BitCount, 1); } else if (column.CompressionType == CompressionType.PalletArray) { column.ArraySize = Math.Max(column.Cardinality, 1); } ColumnMeta.Add(column); } // Pallet values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray) { int elements = (int)ColumnMeta[i].AdditionalDataSize / 4; int cardinality = Math.Max(ColumnMeta[i].Cardinality, 1); ColumnMeta[i].PalletValues = new List <byte[]>(); for (int j = 0; j < elements / cardinality; j++) { ColumnMeta[i].PalletValues.Add(dbReader.ReadBytes(cardinality * 4)); } } } // Sparse values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Sparse) { ColumnMeta[i].SparseValues = new Dictionary <int, byte[]>(); for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++) { ColumnMeta[i].SparseValues[dbReader.ReadInt32()] = dbReader.ReadBytes(4); } } } // Relationships if (RelationshipDataSize > 0) { RelationShipData = new RelationShipData() { Records = dbReader.ReadUInt32(), MinId = dbReader.ReadUInt32(), MaxId = dbReader.ReadUInt32(), Entries = new List <RelationShipEntry>() }; for (int i = 0; i < RelationShipData.Records; i++) { RelationShipData.Entries.Add(new RelationShipEntry(dbReader.ReadUInt32(), dbReader.ReadUInt32())); } FieldStructure.Add(new FieldStructureEntry(0, 0)); ColumnMeta.Add(new ColumnStructureEntry()); } // Record Data BitStream bitStream = new BitStream(recordData); for (int i = 0; i < RecordCount; i++) { int id = 0; if (HasOffsetTable && HasIndexTable) { id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; if (CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) // ignore duplicates { continue; } dbReader.BaseStream.Position = map.Item1; byte[] data = dbReader.ReadBytes(map.Item2); IEnumerable <byte> recordbytes = BitConverter.GetBytes(id).Concat(data); if (RelationShipData != null) { byte[] relation = BitConverter.GetBytes(RelationShipData.Entries.First(x => x.Index == i).Id); recordbytes = recordbytes.Concat(relation); } CopyTable.Add(id, recordbytes.ToArray()); if (Copies.ContainsKey(id)) { foreach (var copy in Copies[id]) { CopyTable.Add(copy, BitConverter.GetBytes(copy).Concat(data).ToArray()); } } } else { bitStream.Seek(i * RecordSize, 0); int idOffset = 0; List <byte> data = new List <byte>(); if (HasIndexTable) { id = m_indexes[i]; data.AddRange(BitConverter.GetBytes(id)); } for (int f = 0; f < FieldCount; f++) { int bitOffset = ColumnMeta[f].BitOffset; int bitWidth = ColumnMeta[f].BitWidth; int cardinality = ColumnMeta[f].Cardinality; uint palletIndex; switch (ColumnMeta[f].CompressionType) { case CompressionType.None: int bitSize = FieldStructure[f].BitCount; if (!HasIndexTable && f == IdIndex) { idOffset = data.Count; id = (int)bitStream.ReadUInt32(bitSize); // always read Ids as ints data.AddRange(BitConverter.GetBytes(id)); } else { for (int x = 0; x < ColumnMeta[f].ArraySize; x++) { data.AddRange(bitStream.ReadBytesPadded(bitSize)); } } break; case CompressionType.Immediate: if (!HasIndexTable && f == IdIndex) { idOffset = data.Count; id = (int)bitStream.ReadUInt32(bitWidth); // always read Ids as ints data.AddRange(BitConverter.GetBytes(id)); continue; } else { data.AddRange(bitStream.ReadBytesPadded(bitWidth)); } break; case CompressionType.Sparse: if (ColumnMeta[f].SparseValues.TryGetValue(id, out byte[] valBytes))