protected void RemoveBitLimits() { if (HasOffsetTable) { return; } int c = HasIndexTable ? 1 : 0; int cm = ColumnMeta.Count - (RelationShipData != null ? 1 : 0); var skipType = new HashSet <CompressionType>(new[] { CompressionType.None, CompressionType.Sparse }); for (int i = c; i < cm; i++) { ColumnStructureEntry col = ColumnMeta[i]; CompressionType type = col.CompressionType; int oldsize = col.BitWidth; var newsize = (ushort)(columnSizes[c] * 8); c += col.ArraySize; if (skipType.Contains(col.CompressionType) || newsize == oldsize) { continue; } col.BitWidth = col.Size = newsize; for (int x = i + 1; x < cm; x++) { if (skipType.Contains(ColumnMeta[x].CompressionType)) { continue; } ColumnMeta[x].RecordOffset += (ushort)(newsize - oldsize); ColumnMeta[x].BitOffset = ColumnMeta[x].RecordOffset - PackedDataOffset * 8; } } RecordSize = (uint)((ColumnMeta.Sum(x => x.Size) + 7) / 8); }
static Dictionary <int, byte[]> ReadData(BinaryReader reader) { Dictionary <int, byte[]> CopyTable = new Dictionary <int, byte[]>(); List <Tuple <int, short> > offsetmap = new List <Tuple <int, short> >(); Dictionary <int, OffsetDuplicate> firstindex = new Dictionary <int, OffsetDuplicate>(); Dictionary <int, int> OffsetDuplicates = new Dictionary <int, int>(); Dictionary <int, List <int> > Copies = new Dictionary <int, List <int> >(); byte[] recordData; if (Header.HasOffsetTable()) { recordData = reader.ReadBytes((int)(Header.OffsetTableOffset - 84 - 4 * Header.FieldCount)); } else { recordData = reader.ReadBytes((int)(Header.RecordCount * Header.RecordSize)); Array.Resize(ref recordData, recordData.Length + 8); } if (Header.StringTableSize != 0) { // string data StringTable = new Dictionary <int, string>(); for (int i = 0; i < Header.StringTableSize;) { long oldPos = reader.BaseStream.Position; StringTable[i] = reader.ReadCString(); i += (int)(reader.BaseStream.Position - oldPos); } } int[] m_indexes = null; // OffsetTable if (Header.HasOffsetTable() && Header.OffsetTableOffset > 0) { reader.BaseStream.Position = Header.OffsetTableOffset; for (int i = 0; i < (Header.MaxId - Header.MinId + 1); i++) { int offset = reader.ReadInt32(); short length = reader.ReadInt16(); if (offset == 0 || length == 0) { continue; } // special case, may contain duplicates in the offset map that we don't want if (Header.CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); } else { OffsetDuplicates.Add(Header.MinId + i, firstindex[offset].VisibleIndex); } } offsetmap.Add(new Tuple <int, short>(offset, length)); } } // IndexTable if (Header.HasIndexTable()) { m_indexes = reader.ReadArray <int>(Header.RecordCount); } // Copytable if (Header.CopyTableSize > 0) { long end = reader.BaseStream.Position + Header.CopyTableSize; while (reader.BaseStream.Position < end) { int id = reader.ReadInt32(); int idcopy = reader.ReadInt32(); if (!Copies.ContainsKey(idcopy)) { Copies.Add(idcopy, new List <int>()); } Copies[idcopy].Add(id); } } // ColumnMeta ColumnMeta = new List <ColumnStructureEntry>(); if (Header.ColumnMetaSize != 0) { for (int i = 0; i < Header.FieldCount; i++) { var column = new ColumnStructureEntry() { RecordOffset = reader.ReadUInt16(), Size = reader.ReadUInt16(), AdditionalDataSize = reader.ReadUInt32(), // size of pallet / sparse values CompressionType = (DB2ColumnCompression)reader.ReadUInt32(), BitOffset = reader.ReadInt32(), BitWidth = reader.ReadInt32(), Cardinality = reader.ReadInt32() }; // preload arraysizes if (column.CompressionType == DB2ColumnCompression.None) { column.ArraySize = Math.Max(column.Size / FieldStructure[i].BitCount, 1); } else if (column.CompressionType == DB2ColumnCompression.PalletArray) { column.ArraySize = Math.Max(column.Cardinality, 1); } ColumnMeta.Add(column); } } // Pallet values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == DB2ColumnCompression.Pallet || ColumnMeta[i].CompressionType == DB2ColumnCompression.PalletArray) { int elements = (int)ColumnMeta[i].AdditionalDataSize / 4; int cardinality = Math.Max(ColumnMeta[i].Cardinality, 1); ColumnMeta[i].PalletValues = new List <byte[]>(); for (int j = 0; j < elements / cardinality; j++) { ColumnMeta[i].PalletValues.Add(reader.ReadBytes(cardinality * 4)); } } } // Sparse values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == DB2ColumnCompression.CommonData) { ColumnMeta[i].SparseValues = new Dictionary <int, byte[]>(); for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++) { ColumnMeta[i].SparseValues[reader.ReadInt32()] = reader.ReadBytes(4); } } } // Relationships if (Header.RelationshipDataSize > 0) { RelationShipData = new RelationShipData() { Records = reader.ReadUInt32(), MinId = reader.ReadUInt32(), MaxId = reader.ReadUInt32(), Entries = new Dictionary <uint, byte[]>() }; for (int i = 0; i < RelationShipData.Records; i++) { byte[] foreignKey = reader.ReadBytes(4); uint index = reader.ReadUInt32(); // has duplicates just like the copy table does... why? if (!RelationShipData.Entries.ContainsKey(index)) { RelationShipData.Entries.Add(index, foreignKey); } } } // Record Data for (int i = 0; i < Header.RecordCount; i++) { int id = 0; if (Header.HasOffsetTable() && Header.HasIndexTable()) { id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; if (Header.CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) // ignore duplicates { continue; } reader.BaseStream.Position = map.Item1; byte[] data = reader.ReadBytes(map.Item2); IEnumerable <byte> recordbytes = data; // append relationship id if (RelationShipData != null) { // seen cases of missing indicies if (RelationShipData.Entries.TryGetValue((uint)i, out byte[] foreignData))
public override void ReadHeader(ref BinaryReader dbReader, string signature) { ReadBaseHeader(ref dbReader, signature); TableHash = dbReader.ReadUInt32(); LayoutHash = dbReader.ReadInt32(); MinId = dbReader.ReadInt32(); MaxId = dbReader.ReadInt32(); Locale = dbReader.ReadInt32(); Flags = (HeaderFlags)dbReader.ReadUInt16(); IdIndex = dbReader.ReadUInt16(); TotalFieldSize = dbReader.ReadUInt32(); PackedDataOffset = dbReader.ReadInt32(); RelationshipCount = dbReader.ReadUInt32(); ColumnMetadataSize = dbReader.ReadInt32(); SparseDataSize = dbReader.ReadInt32(); PalletDataSize = dbReader.ReadInt32(); SectionCount = dbReader.ReadInt32(); // TODO convert to array when the time comes Unknown1 = dbReader.ReadInt32(); Unknown2 = dbReader.ReadInt32(); RecordDataOffset = dbReader.ReadInt32(); RecordDataRowCount = dbReader.ReadInt32(); RecordDataStringSize = dbReader.ReadInt32(); CopyTableSize = dbReader.ReadInt32(); OffsetTableOffset = dbReader.ReadInt32(); IndexSize = dbReader.ReadInt32(); RelationshipDataSize = dbReader.ReadInt32(); if (RecordCount == 0 || FieldCount == 0) { return; } //Gather field structures FieldStructure = new List <FieldStructureEntry>(); for (int i = 0; i < FieldCount; i++) { var field = new FieldStructureEntry(dbReader.ReadInt16(), dbReader.ReadUInt16()); FieldStructure.Add(field); } // ColumnMeta ColumnMeta = new List <ColumnStructureEntry>(); for (int i = 0; i < FieldCount; i++) { var column = new ColumnStructureEntry() { RecordOffset = dbReader.ReadUInt16(), Size = dbReader.ReadUInt16(), AdditionalDataSize = dbReader.ReadUInt32(), // size of pallet / sparse values CompressionType = (CompressionType)dbReader.ReadUInt32(), BitOffset = dbReader.ReadInt32(), BitWidth = dbReader.ReadInt32(), Cardinality = dbReader.ReadInt32() }; // preload arraysizes if (column.CompressionType == CompressionType.None) { column.ArraySize = Math.Max(column.Size / FieldStructure[i].BitCount, 1); } else if (column.CompressionType == CompressionType.PalletArray) { column.ArraySize = Math.Max(column.Cardinality, 1); } ColumnMeta.Add(column); } // Pallet values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray) { int elements = (int)ColumnMeta[i].AdditionalDataSize / 4; int cardinality = Math.Max(ColumnMeta[i].Cardinality, 1); ColumnMeta[i].PalletValues = new List <byte[]>(); for (int j = 0; j < elements / cardinality; j++) { ColumnMeta[i].PalletValues.Add(dbReader.ReadBytes(cardinality * 4)); } } } // Sparse values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Sparse) { ColumnMeta[i].SparseValues = new Dictionary <int, byte[]>(); for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++) { ColumnMeta[i].SparseValues[dbReader.ReadInt32()] = dbReader.ReadBytes(4); } } } // RecordData recordData = dbReader.ReadBytes((int)(RecordCount * RecordSize)); Array.Resize(ref recordData, recordData.Length + 8); Flags &= ~HeaderFlags.SecondIndex; // appears to be obsolete now }
public new Dictionary <int, byte[]> ReadOffsetData(BinaryReader dbReader, long pos) { var CopyTable = new Dictionary <int, byte[]>(); var offsetmap = new List <Tuple <int, short> >(); var firstindex = new Dictionary <int, OffsetDuplicate>(); var OffsetDuplicates = new Dictionary <int, int>(); var Copies = new Dictionary <int, List <int> >(); int[] m_indexes = null; // OffsetTable if (HasOffsetTable && OffsetTableOffset > 0) { dbReader.BaseStream.Position = OffsetTableOffset; for (var i = 0; i < MaxId - MinId + 1; i++) { int offset = dbReader.ReadInt32(); short length = dbReader.ReadInt16(); if (offset == 0 || length == 0) { continue; } // special case, may contain duplicates in the offset map that we don't want if (CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); } else { OffsetDuplicates.Add(MinId + i, firstindex[offset].VisibleIndex); continue; } } offsetmap.Add(new Tuple <int, short>(offset, length)); } } // IndexTable if (HasIndexTable) { m_indexes = new int[RecordCount]; for (var i = 0; i < RecordCount; i++) { m_indexes[i] = dbReader.ReadInt32(); } } // Copytable if (CopyTableSize > 0) { long end = dbReader.BaseStream.Position + CopyTableSize; while (dbReader.BaseStream.Position < end) { int id = dbReader.ReadInt32(); int idcopy = dbReader.ReadInt32(); if (!Copies.ContainsKey(idcopy)) { Copies.Add(idcopy, new List <int>()); } Copies[idcopy].Add(id); } } // ColumnMeta ColumnMeta = new List <ColumnStructureEntry>(); for (var i = 0; i < FieldCount; i++) { ColumnStructureEntry column = new ColumnStructureEntry { RecordOffset = dbReader.ReadUInt16(), Size = dbReader.ReadUInt16(), AdditionalDataSize = dbReader.ReadUInt32(), // size of pallet / sparse values CompressionType = (CompressionType)dbReader.ReadUInt32(), BitOffset = dbReader.ReadInt32(), BitWidth = dbReader.ReadInt32(), Cardinality = dbReader.ReadInt32() }; // preload arraysizes if (column.CompressionType == CompressionType.None) { column.ArraySize = Math.Max(column.Size / FieldStructure[i].BitCount, 1); } else if (column.CompressionType == CompressionType.PalletArray) { column.ArraySize = Math.Max(column.Cardinality, 1); } ColumnMeta.Add(column); } // Pallet values for (var i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray) { int elements = (int)ColumnMeta[i].AdditionalDataSize / 4; int cardinality = Math.Max(ColumnMeta[i].Cardinality, 1); ColumnMeta[i].PalletValues = new List <byte[]>(); for (var j = 0; j < elements / cardinality; j++) { ColumnMeta[i].PalletValues.Add(dbReader.ReadBytes(cardinality * 4)); } } } // Sparse values for (var i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Sparse) { ColumnMeta[i].SparseValues = new Dictionary <int, byte[]>(); for (var j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++) { ColumnMeta[i].SparseValues[dbReader.ReadInt32()] = dbReader.ReadBytes(4); } } } // Relationships if (RelationshipDataSize > 0) { RelationShipData = new RelationShipData { Records = dbReader.ReadUInt32(), MinId = dbReader.ReadUInt32(), MaxId = dbReader.ReadUInt32(), Entries = new Dictionary <uint, byte[]>() }; for (var i = 0; i < RelationShipData.Records; i++) { byte[] foreignKey = dbReader.ReadBytes(4); uint index = dbReader.ReadUInt32(); // has duplicates just like the copy table does... why? if (!RelationShipData.Entries.ContainsKey(index)) { RelationShipData.Entries.Add(index, foreignKey); } } FieldStructure.Add(new FieldStructureEntry(0, 0)); ColumnMeta.Add(new ColumnStructureEntry()); } // Record Data BitStream bitStream = new(recordData); for (var i = 0; i < RecordCount; i++) { var id = 0; if (HasOffsetTable && HasIndexTable) { id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; if (CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) //Ignore duplicates { continue; } dbReader.BaseStream.Position = map.Item1; byte[] data = dbReader.ReadBytes(map.Item2); var recordbytes = BitConverter.GetBytes(id).Concat(data); // append relationship id if (RelationShipData != null) { // seen cases of missing indicies if (RelationShipData.Entries.TryGetValue((uint)i, out byte[] foreignData))
public new Dictionary <int, byte[]> ReadOffsetData(BinaryReader dbReader, long pos) { Dictionary <int, byte[]> CopyTable = new Dictionary <int, byte[]>(); List <Tuple <int, short> > offsetmap = new List <Tuple <int, short> >(); Dictionary <int, OffsetDuplicate> firstindex = new Dictionary <int, OffsetDuplicate>(); Dictionary <int, int> OffsetDuplicates = new Dictionary <int, int>(); Dictionary <int, List <int> > Copies = new Dictionary <int, List <int> >(); int[] m_indexes = null; // OffsetTable if (HasOffsetTable && OffsetTableOffset > 0) { dbReader.BaseStream.Position = OffsetTableOffset; for (int i = 0; i < (MaxId - MinId + 1); i++) { int offset = dbReader.ReadInt32(); short length = dbReader.ReadInt16(); if (offset == 0 || length == 0) { continue; } // special case, may contain duplicates in the offset map that we don't want if (CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); } else { OffsetDuplicates.Add(MinId + i, firstindex[offset].VisibleIndex); } } offsetmap.Add(new Tuple <int, short>(offset, length)); } } // IndexTable if (HasIndexTable) { m_indexes = new int[RecordCount]; for (int i = 0; i < RecordCount; i++) { m_indexes[i] = dbReader.ReadInt32(); } } // Copytable if (CopyTableSize > 0) { long end = dbReader.BaseStream.Position + CopyTableSize; while (dbReader.BaseStream.Position < end) { int id = dbReader.ReadInt32(); int idcopy = dbReader.ReadInt32(); if (!Copies.ContainsKey(idcopy)) { Copies.Add(idcopy, new List <int>()); } Copies[idcopy].Add(id); } } // ColumnMeta ColumnMeta = new List <ColumnStructureEntry>(); for (int i = 0; i < FieldCount; i++) { var column = new ColumnStructureEntry() { RecordOffset = dbReader.ReadUInt16(), Size = dbReader.ReadUInt16(), AdditionalDataSize = dbReader.ReadUInt32(), // size of pallet / sparse values CompressionType = (CompressionType)dbReader.ReadUInt32(), BitOffset = dbReader.ReadInt32(), BitWidth = dbReader.ReadInt32(), Cardinality = dbReader.ReadInt32() }; // preload arraysizes if (column.CompressionType == CompressionType.None) { column.ArraySize = Math.Max(column.Size / FieldStructure[i].BitCount, 1); } else if (column.CompressionType == CompressionType.PalletArray) { column.ArraySize = Math.Max(column.Cardinality, 1); } ColumnMeta.Add(column); } // Pallet values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray) { int elements = (int)ColumnMeta[i].AdditionalDataSize / 4; int cardinality = Math.Max(ColumnMeta[i].Cardinality, 1); ColumnMeta[i].PalletValues = new List <byte[]>(); for (int j = 0; j < elements / cardinality; j++) { ColumnMeta[i].PalletValues.Add(dbReader.ReadBytes(cardinality * 4)); } } } // Sparse values for (int i = 0; i < ColumnMeta.Count; i++) { if (ColumnMeta[i].CompressionType == CompressionType.Sparse) { ColumnMeta[i].SparseValues = new Dictionary <int, byte[]>(); for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++) { ColumnMeta[i].SparseValues[dbReader.ReadInt32()] = dbReader.ReadBytes(4); } } } // Relationships if (RelationshipDataSize > 0) { RelationShipData = new RelationShipData() { Records = dbReader.ReadUInt32(), MinId = dbReader.ReadUInt32(), MaxId = dbReader.ReadUInt32(), Entries = new List <RelationShipEntry>() }; for (int i = 0; i < RelationShipData.Records; i++) { RelationShipData.Entries.Add(new RelationShipEntry(dbReader.ReadUInt32(), dbReader.ReadUInt32())); } FieldStructure.Add(new FieldStructureEntry(0, 0)); ColumnMeta.Add(new ColumnStructureEntry()); } // Record Data BitStream bitStream = new BitStream(recordData); for (int i = 0; i < RecordCount; i++) { int id = 0; if (HasOffsetTable && HasIndexTable) { id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; if (CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) // ignore duplicates { continue; } dbReader.BaseStream.Position = map.Item1; byte[] data = dbReader.ReadBytes(map.Item2); IEnumerable <byte> recordbytes = BitConverter.GetBytes(id).Concat(data); if (RelationShipData != null) { byte[] relation = BitConverter.GetBytes(RelationShipData.Entries.First(x => x.Index == i).Id); recordbytes = recordbytes.Concat(relation); } CopyTable.Add(id, recordbytes.ToArray()); if (Copies.ContainsKey(id)) { foreach (var copy in Copies[id]) { CopyTable.Add(copy, BitConverter.GetBytes(copy).Concat(data).ToArray()); } } } else { bitStream.Seek(i * RecordSize, 0); int idOffset = 0; List <byte> data = new List <byte>(); if (HasIndexTable) { id = m_indexes[i]; data.AddRange(BitConverter.GetBytes(id)); } for (int f = 0; f < FieldCount; f++) { int bitOffset = ColumnMeta[f].BitOffset; int bitWidth = ColumnMeta[f].BitWidth; int cardinality = ColumnMeta[f].Cardinality; uint palletIndex; switch (ColumnMeta[f].CompressionType) { case CompressionType.None: int bitSize = FieldStructure[f].BitCount; if (!HasIndexTable && f == IdIndex) { idOffset = data.Count; id = (int)bitStream.ReadUInt32(bitSize); // always read Ids as ints data.AddRange(BitConverter.GetBytes(id)); } else { for (int x = 0; x < ColumnMeta[f].ArraySize; x++) { data.AddRange(bitStream.ReadBytesPadded(bitSize)); } } break; case CompressionType.Immediate: if (!HasIndexTable && f == IdIndex) { idOffset = data.Count; id = (int)bitStream.ReadUInt32(bitWidth); // always read Ids as ints data.AddRange(BitConverter.GetBytes(id)); continue; } else { data.AddRange(bitStream.ReadBytesPadded(bitWidth)); } break; case CompressionType.Sparse: if (ColumnMeta[f].SparseValues.TryGetValue(id, out byte[] valBytes))