public void file_size_should_be_greater_or_equal_after_adding_midpoints(byte version, int minDepth) { for (long numIndexEntries = 0; numIndexEntries < MAX_INDEX_ENTRIES; numIndexEntries += getIncrement(numIndexEntries)) { string testCase = $"numIndexEntries: {numIndexEntries}, minDepth: {minDepth}, version: {version}"; var midpointCount = PTable.GetRequiredMidpointCountCached(numIndexEntries, version, minDepth); if (version < PTableVersions.IndexV4) { Assert.AreEqual(0, midpointCount, testCase); } else { Assert.GreaterOrEqual(midpointCount, Math.Min(1 << Math.Min(MAX_DEPTH, minDepth), numIndexEntries), testCase); Assert.LessOrEqual(midpointCount, Math.Max(2, numIndexEntries), testCase); } long fileSizeUpToIndexEntries = PTable.GetFileSizeUpToIndexEntries(numIndexEntries, version); long fileSizeUpToMidpointEntries = PTable.GetFileSizeUpToMidpointEntries(fileSizeUpToIndexEntries, midpointCount, version); if (version < PTableVersions.IndexV4) { Assert.AreEqual(fileSizeUpToIndexEntries, fileSizeUpToMidpointEntries, testCase); } else { Assert.GreaterOrEqual(fileSizeUpToMidpointEntries, fileSizeUpToIndexEntries, testCase); } } }
private void construct_same_midpoint_indexes_for_any_combination_of_params(int maxIndexEntries) { for (var numIndexEntries = 0; numIndexEntries < maxIndexEntries; numIndexEntries++) { for (var depth = 0; depth < 20; depth++) { var requiredMidpointsCount = PTable.GetRequiredMidpointCountCached(numIndexEntries, _ptableVersion, depth); List <long> requiredMidpoints = new List <long>(); for (var k = 0; k < requiredMidpointsCount; k++) { var index = PTable.GetMidpointIndex(k, numIndexEntries, requiredMidpointsCount); requiredMidpoints.Add(index); } List <long> calculatedMidpoints = new List <long>(); for (var k = 0; k < numIndexEntries; k++) { if (PTable.IsMidpointIndex(k, numIndexEntries, requiredMidpointsCount)) { calculatedMidpoints.Add(k); } } if (numIndexEntries == 1 && calculatedMidpoints.Count == 1) { calculatedMidpoints.Add(calculatedMidpoints[0]); } if (requiredMidpoints.Count != calculatedMidpoints.Count) { Log.Error( "Midpoint count mismatch for numIndexEntries: {0}, depth:{1} - Expected {2}, Found {3}", numIndexEntries, depth, requiredMidpoints.Count, calculatedMidpoints.Count); } Assert.AreEqual(requiredMidpoints.Count, calculatedMidpoints.Count); for (var i = 0; i < requiredMidpoints.Count; i++) { if (requiredMidpoints[i] != calculatedMidpoints[i]) { Log.Error( "Midpoint mismatch at index {0} for numIndexEntries: {1}, depth:{2} - Expected {3}, Found {4}", i, numIndexEntries, depth, requiredMidpoints[i], calculatedMidpoints[i]); } Assert.AreEqual(requiredMidpoints[i], calculatedMidpoints[i]); } } } }
public void the_file_gets_created_and_can_be_read() { var indexEntrySize = PTable.IndexEntryV4Size; if (_ptableVersion == PTableVersions.IndexV1) { indexEntrySize = PTable.IndexEntryV1Size; } else if (_ptableVersion == PTableVersions.IndexV2) { indexEntrySize = PTable.IndexEntryV2Size; } else if (_ptableVersion == PTableVersions.IndexV3) { indexEntrySize = PTable.IndexEntryV3Size; } var table = new HashListMemTable(_ptableVersion, maxSize: 10); table.Add(0x010100000000, 0x0001, 0x0001); table.Add(0x010500000000, 0x0001, 0x0002); table.Add(0x010200000000, 0x0001, 0x0003); table.Add(0x010200000000, 0x0002, 0x0003); using (var sstable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify, useBloomFilter: true)) { var fileinfo = new FileInfo(Filename); var midpointsCached = PTable.GetRequiredMidpointCountCached(4, _ptableVersion); Assert.AreEqual( PTableHeader.Size + 4 * indexEntrySize + midpointsCached * indexEntrySize + PTableFooter.GetSize(_ptableVersion) + PTable.MD5Size, fileinfo.Length); var items = sstable.IterateAllInOrder().ToList(); Assert.AreEqual(GetHash(0x010500000000), items[0].Stream); Assert.AreEqual(0x0001, items[0].Version); Assert.AreEqual(GetHash(0x010200000000), items[1].Stream); Assert.AreEqual(0x0002, items[1].Version); Assert.AreEqual(GetHash(0x010200000000), items[2].Stream); Assert.AreEqual(0x0001, items[2].Version); Assert.AreEqual(GetHash(0x010100000000), items[3].Stream); Assert.AreEqual(0x0001, items[3].Version); var stream = (ulong)0x010200000000; Assert.True(sstable.TryGetLatestEntry(stream, out var entry)); Assert.AreEqual(GetHash(stream), entry.Stream); Assert.AreEqual(0x0002, entry.Version); Assert.AreEqual(0x0003, entry.Position); } }
public void for_a_fixed_number_of_index_entries_and_increasing_min_depth_midpoint_count_should_increase_monotonically() { for (long numIndexEntries = 0; numIndexEntries < MAX_INDEX_ENTRIES; numIndexEntries += getIncrement(numIndexEntries)) { long prevMidpointCount = 0L; for (int minDepth = 0; minDepth <= MAX_TEST_DEPTH; minDepth++) { string testCase = $"numIndexEntries: {numIndexEntries}, minDepth: {minDepth}"; var midpointCount = PTable.GetRequiredMidpointCountCached(numIndexEntries, PTableVersions.IndexV4, minDepth); Assert.GreaterOrEqual(midpointCount, prevMidpointCount, testCase); Assert.GreaterOrEqual(midpointCount, Math.Min(1 << Math.Min(MAX_DEPTH, minDepth), numIndexEntries), testCase); Assert.LessOrEqual(midpointCount, Math.Max(2, numIndexEntries), testCase); prevMidpointCount = midpointCount; } } }
public void the_correct_midpoints_are_cached() { PTable.Midpoint[] midpoints = _newtable.GetMidPoints(); var requiredMidpoints = PTable.GetRequiredMidpointCountCached(_newtable.Count, _ptableVersion); Assert.AreEqual(requiredMidpoints, midpoints.Length); var position = 0; foreach (var item in _newtable.IterateAllInOrder()) { if (PTable.IsMidpointIndex(position, _newtable.Count, requiredMidpoints)) { Assert.AreEqual(item.Stream, midpoints[position].Key.Stream); Assert.AreEqual(item.Version, midpoints[position].Key.Version); Assert.AreEqual(position, midpoints[position].ItemIndex); position++; } } }
public void midpoints_are_cached_in_ptable_footer() { var numIndexEntries = 14; var requiredMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries, PTableVersions.IndexV4); var newTableFileCopy = GetTempFilePath(); File.Copy(_newtableFile, newTableFileCopy); using (var filestream = File.Open(newTableFileCopy, FileMode.Open, FileAccess.Read)) { var footerSize = PTableFooter.GetSize(PTableVersions.IndexV4); Assert.AreEqual(filestream.Length, PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size + footerSize + PTable.MD5Size); filestream.Seek(PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size, SeekOrigin.Begin); var ptableFooter = PTableFooter.FromStream(filestream); Assert.AreEqual(FileType.PTableFile, ptableFooter.FileType); Assert.AreEqual(PTableVersions.IndexV4, ptableFooter.Version); Assert.AreEqual(requiredMidpoints, ptableFooter.NumMidpointsCached); } }
public void the_file_gets_created() { var indexEntrySize = PTable.IndexEntryV4Size; if (_ptableVersion == PTableVersions.IndexV1) { indexEntrySize = PTable.IndexEntryV1Size; } else if (_ptableVersion == PTableVersions.IndexV2) { indexEntrySize = PTable.IndexEntryV2Size; } else if (_ptableVersion == PTableVersions.IndexV3) { indexEntrySize = PTable.IndexEntryV3Size; } var table = new HashListMemTable(_ptableVersion, maxSize: 10); table.Add(0x010100000000, 0x0001, 0x0001); table.Add(0x010500000000, 0x0001, 0x0002); table.Add(0x010200000000, 0x0001, 0x0003); table.Add(0x010200000000, 0x0002, 0x0003); using (var sstable = PTable.FromMemtable(table, Filename, skipIndexVerify: _skipIndexVerify)) { var fileinfo = new FileInfo(Filename); var midpointsCached = PTable.GetRequiredMidpointCountCached(4, _ptableVersion); Assert.AreEqual( PTableHeader.Size + 4 * indexEntrySize + midpointsCached * indexEntrySize + PTableFooter.GetSize(_ptableVersion) + PTable.MD5Size, fileinfo.Length); var items = sstable.IterateAllInOrder().ToList(); Assert.AreEqual(GetHash(0x010500000000), items[0].Stream); Assert.AreEqual(0x0001, items[0].Version); Assert.AreEqual(GetHash(0x010200000000), items[1].Stream); Assert.AreEqual(0x0002, items[1].Version); Assert.AreEqual(GetHash(0x010200000000), items[2].Stream); Assert.AreEqual(0x0001, items[2].Version); Assert.AreEqual(GetHash(0x010100000000), items[3].Stream); Assert.AreEqual(0x0001, items[3].Version); } }
private void CorruptPTableFile(string ptableFile, byte version, string corruptionType) { int indexEntrySize = 0; if (version == PTableVersions.IndexV1) { indexEntrySize = PTable.IndexEntryV1Size; } else if (version == PTableVersions.IndexV2) { indexEntrySize = PTable.IndexEntryV2Size; } else if (version == PTableVersions.IndexV3) { indexEntrySize = PTable.IndexEntryV3Size; } else if (version == PTableVersions.IndexV4) { indexEntrySize = PTable.IndexEntryV4Size; } int indexEntryKeySize = 0; if (version == PTableVersions.IndexV1) { indexEntryKeySize = PTable.IndexKeyV1Size; } else if (version == PTableVersions.IndexV2) { indexEntryKeySize = PTable.IndexKeyV2Size; } else if (version == PTableVersions.IndexV3) { indexEntryKeySize = PTable.IndexKeyV3Size; } else if (version == PTableVersions.IndexV4) { indexEntryKeySize = PTable.IndexKeyV4Size; } uint numMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries, version); byte[] data = new byte[255]; using (FileStream stream = File.Open(ptableFile, FileMode.Open)) { if (corruptionType == "footerFileType") { stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize, SeekOrigin.Begin); data[0] = 0xFF; stream.Write(data, 0, 1); } else if (corruptionType == "footerVersion") { stream.Seek( PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 1, SeekOrigin.Begin); data[0] = 0xFF; stream.Write(data, 0, 1); } else if (corruptionType == "negativeIndexEntriesSize") { stream.Seek( PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2, SeekOrigin.Begin); data[0] = 0xFF; data[1] = 0xFF; data[2] = 0xFF; data[3] = 0xFF; stream.Write(data, 0, 4); } else if (corruptionType == "notMultipleIndexEntrySize") { var footerPosition = PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize; stream.Seek(footerPosition, SeekOrigin.Begin); var buffer = new byte[4096]; int read = stream.Read(buffer, 0, 4096); //insert a byte stream.Seek(footerPosition, SeekOrigin.Begin); data[0] = 0xFF; stream.Write(data, 0, 1); stream.Write(buffer, 0, read); } else if (corruptionType == "lessThan2Midpoints") { stream.Seek( PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2, SeekOrigin.Begin); //1 midpoint data[0] = 0x01; data[1] = 0x00; data[2] = 0x00; data[3] = 0x00; stream.Write(data, 0, 4); } else if (corruptionType == "moreMidpointsThanIndexEntries") { stream.Seek( PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2, SeekOrigin.Begin); //change number of midpoints to 1 more than actual number to trigger the condition var x = numMidpoints + 1; for (int i = 0; i < 4; i++) { data[i] = (byte)(x & 0xFF); x >>= 8; } stream.Write(data, 0, 4); } else if (corruptionType == "zeroOutMiddleEntries") { //zeroes out middle index entries - useful for binary search tests List <int> indexEntriesToCorrupt = new List <int>(); indexEntriesToCorrupt.Add(numIndexEntries / 2 - 1); indexEntriesToCorrupt.Add(numIndexEntries / 2); foreach (int entry in indexEntriesToCorrupt) { stream.Seek(PTableHeader.Size + entry * indexEntrySize, SeekOrigin.Begin); //modify one of the index entry hashes/version stream.Write(data, 0, indexEntryKeySize); if (version >= PTableVersions.IndexV4) { //modify one of the midpoint entry hashes/version stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + entry * indexEntrySize, SeekOrigin.Begin); stream.Write(data, 0, indexEntryKeySize); } } } else if (corruptionType == "maxOutMiddleEntries") { //maxes out middle index entries - useful for binary search tests List <int> indexEntriesToCorrupt = new List <int>(); indexEntriesToCorrupt.Add(numIndexEntries / 2 - 1); indexEntriesToCorrupt.Add(numIndexEntries / 2); for (int i = 0; i < indexEntryKeySize; i++) { data[i] = 0xFF; } foreach (int entry in indexEntriesToCorrupt) { stream.Seek(PTableHeader.Size + entry * indexEntrySize, SeekOrigin.Begin); //modify one of the index entry hashes/version stream.Write(data, 0, indexEntryKeySize); if (version >= PTableVersions.IndexV4) { //modify one of the midpoint entry hashes/version stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + entry * indexEntrySize, SeekOrigin.Begin); stream.Write(data, 0, indexEntryKeySize); } } } else if (corruptionType == "midpointItemIndexesNotAscendingOrder") { if (version >= PTableVersions.IndexV4) { //modify one of the midpoint item indexes stream.Seek( PTableHeader.Size + numIndexEntries * indexEntrySize + 3 * indexEntrySize - sizeof(long), SeekOrigin.Begin); stream.Write(data, 0, sizeof(long)); } } } }
public void correct_number_of_midpoints_are_loaded() { Assert.AreEqual(_newtable.GetMidPoints().Length, PTable.GetRequiredMidpointCountCached(14, PTableVersions.IndexV4)); }