public void file_size_should_be_greater_or_equal_after_adding_midpoints(byte version, int minDepth)
{
for (long numIndexEntries = 0; numIndexEntries < MAX_INDEX_ENTRIES; numIndexEntries += getIncrement(numIndexEntries))
{
string testCase = $"numIndexEntries: {numIndexEntries}, minDepth: {minDepth}, version: {version}";
var midpointCount = PTable.GetRequiredMidpointCountCached(numIndexEntries, version, minDepth);
if (version < PTableVersions.IndexV4)
{
Assert.AreEqual(0, midpointCount, testCase);
}
else
{
Assert.GreaterOrEqual(midpointCount, Math.Min(1 << Math.Min(MAX_DEPTH, minDepth), numIndexEntries), testCase);
Assert.LessOrEqual(midpointCount, Math.Max(2, numIndexEntries), testCase);
}
long fileSizeUpToIndexEntries = PTable.GetFileSizeUpToIndexEntries(numIndexEntries, version);
long fileSizeUpToMidpointEntries =
PTable.GetFileSizeUpToMidpointEntries(fileSizeUpToIndexEntries, midpointCount, version);
if (version < PTableVersions.IndexV4)
{
Assert.AreEqual(fileSizeUpToIndexEntries, fileSizeUpToMidpointEntries, testCase);
}
else
{
Assert.GreaterOrEqual(fileSizeUpToMidpointEntries, fileSizeUpToIndexEntries, testCase);
}
}
}
private void construct_same_midpoint_indexes_for_any_combination_of_params(int maxIndexEntries)
{
for (var numIndexEntries = 0; numIndexEntries < maxIndexEntries; numIndexEntries++)
{
for (var depth = 0; depth < 20; depth++)
{
var requiredMidpointsCount =
PTable.GetRequiredMidpointCountCached(numIndexEntries, _ptableVersion, depth);
List <long> requiredMidpoints = new List <long>();
for (var k = 0; k < requiredMidpointsCount; k++)
{
var index = PTable.GetMidpointIndex(k, numIndexEntries, requiredMidpointsCount);
requiredMidpoints.Add(index);
}
List <long> calculatedMidpoints = new List <long>();
for (var k = 0; k < numIndexEntries; k++)
{
if (PTable.IsMidpointIndex(k, numIndexEntries, requiredMidpointsCount))
{
calculatedMidpoints.Add(k);
}
}
if (numIndexEntries == 1 && calculatedMidpoints.Count == 1)
{
calculatedMidpoints.Add(calculatedMidpoints[0]);
}
if (requiredMidpoints.Count != calculatedMidpoints.Count)
{
Log.Error(
"Midpoint count mismatch for numIndexEntries: {0}, depth:{1} - Expected {2}, Found {3}",
numIndexEntries, depth, requiredMidpoints.Count, calculatedMidpoints.Count);
}
Assert.AreEqual(requiredMidpoints.Count, calculatedMidpoints.Count);
for (var i = 0; i < requiredMidpoints.Count; i++)
{
if (requiredMidpoints[i] != calculatedMidpoints[i])
{
Log.Error(
"Midpoint mismatch at index {0} for numIndexEntries: {1}, depth:{2} - Expected {3}, Found {4}",
i, numIndexEntries, depth, requiredMidpoints[i], calculatedMidpoints[i]);
}
Assert.AreEqual(requiredMidpoints[i], calculatedMidpoints[i]);
}
}
}
}
public void the_file_gets_created_and_can_be_read()
{
var indexEntrySize = PTable.IndexEntryV4Size;
if (_ptableVersion == PTableVersions.IndexV1)
{
indexEntrySize = PTable.IndexEntryV1Size;
}
else if (_ptableVersion == PTableVersions.IndexV2)
{
indexEntrySize = PTable.IndexEntryV2Size;
}
else if (_ptableVersion == PTableVersions.IndexV3)
{
indexEntrySize = PTable.IndexEntryV3Size;
}
var table = new HashListMemTable(_ptableVersion, maxSize: 10);
table.Add(0x010100000000, 0x0001, 0x0001);
table.Add(0x010500000000, 0x0001, 0x0002);
table.Add(0x010200000000, 0x0001, 0x0003);
table.Add(0x010200000000, 0x0002, 0x0003);
using (var sstable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault,
skipIndexVerify: _skipIndexVerify,
useBloomFilter: true)) {
var fileinfo = new FileInfo(Filename);
var midpointsCached = PTable.GetRequiredMidpointCountCached(4, _ptableVersion);
Assert.AreEqual(
PTableHeader.Size + 4 * indexEntrySize + midpointsCached * indexEntrySize +
PTableFooter.GetSize(_ptableVersion) + PTable.MD5Size, fileinfo.Length);
var items = sstable.IterateAllInOrder().ToList();
Assert.AreEqual(GetHash(0x010500000000), items[0].Stream);
Assert.AreEqual(0x0001, items[0].Version);
Assert.AreEqual(GetHash(0x010200000000), items[1].Stream);
Assert.AreEqual(0x0002, items[1].Version);
Assert.AreEqual(GetHash(0x010200000000), items[2].Stream);
Assert.AreEqual(0x0001, items[2].Version);
Assert.AreEqual(GetHash(0x010100000000), items[3].Stream);
Assert.AreEqual(0x0001, items[3].Version);
var stream = (ulong)0x010200000000;
Assert.True(sstable.TryGetLatestEntry(stream, out var entry));
Assert.AreEqual(GetHash(stream), entry.Stream);
Assert.AreEqual(0x0002, entry.Version);
Assert.AreEqual(0x0003, entry.Position);
}
}
public void for_a_fixed_number_of_index_entries_and_increasing_min_depth_midpoint_count_should_increase_monotonically()
{
for (long numIndexEntries = 0; numIndexEntries < MAX_INDEX_ENTRIES; numIndexEntries += getIncrement(numIndexEntries))
{
long prevMidpointCount = 0L;
for (int minDepth = 0; minDepth <= MAX_TEST_DEPTH; minDepth++)
{
string testCase = $"numIndexEntries: {numIndexEntries}, minDepth: {minDepth}";
var midpointCount = PTable.GetRequiredMidpointCountCached(numIndexEntries, PTableVersions.IndexV4, minDepth);
Assert.GreaterOrEqual(midpointCount, prevMidpointCount, testCase);
Assert.GreaterOrEqual(midpointCount, Math.Min(1 << Math.Min(MAX_DEPTH, minDepth), numIndexEntries), testCase);
Assert.LessOrEqual(midpointCount, Math.Max(2, numIndexEntries), testCase);
prevMidpointCount = midpointCount;
}
}
}
public void the_correct_midpoints_are_cached()
{
PTable.Midpoint[] midpoints = _newtable.GetMidPoints();
var requiredMidpoints = PTable.GetRequiredMidpointCountCached(_newtable.Count, _ptableVersion);
Assert.AreEqual(requiredMidpoints, midpoints.Length);
var position = 0;
foreach (var item in _newtable.IterateAllInOrder())
{
if (PTable.IsMidpointIndex(position, _newtable.Count, requiredMidpoints))
{
Assert.AreEqual(item.Stream, midpoints[position].Key.Stream);
Assert.AreEqual(item.Version, midpoints[position].Key.Version);
Assert.AreEqual(position, midpoints[position].ItemIndex);
position++;
}
}
}
public void midpoints_are_cached_in_ptable_footer()
{
var numIndexEntries = 14;
var requiredMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries, PTableVersions.IndexV4);
var newTableFileCopy = GetTempFilePath();
File.Copy(_newtableFile, newTableFileCopy);
using (var filestream = File.Open(newTableFileCopy, FileMode.Open, FileAccess.Read))
{
var footerSize = PTableFooter.GetSize(PTableVersions.IndexV4);
Assert.AreEqual(filestream.Length, PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size + footerSize + PTable.MD5Size);
filestream.Seek(PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size, SeekOrigin.Begin);
var ptableFooter = PTableFooter.FromStream(filestream);
Assert.AreEqual(FileType.PTableFile, ptableFooter.FileType);
Assert.AreEqual(PTableVersions.IndexV4, ptableFooter.Version);
Assert.AreEqual(requiredMidpoints, ptableFooter.NumMidpointsCached);
}
}
public void the_file_gets_created()
{
var indexEntrySize = PTable.IndexEntryV4Size;
if (_ptableVersion == PTableVersions.IndexV1)
{
indexEntrySize = PTable.IndexEntryV1Size;
}
else if (_ptableVersion == PTableVersions.IndexV2)
{
indexEntrySize = PTable.IndexEntryV2Size;
}
else if (_ptableVersion == PTableVersions.IndexV3)
{
indexEntrySize = PTable.IndexEntryV3Size;
}
var table = new HashListMemTable(_ptableVersion, maxSize: 10);
table.Add(0x010100000000, 0x0001, 0x0001);
table.Add(0x010500000000, 0x0001, 0x0002);
table.Add(0x010200000000, 0x0001, 0x0003);
table.Add(0x010200000000, 0x0002, 0x0003);
using (var sstable = PTable.FromMemtable(table, Filename, skipIndexVerify: _skipIndexVerify)) {
var fileinfo = new FileInfo(Filename);
var midpointsCached = PTable.GetRequiredMidpointCountCached(4, _ptableVersion);
Assert.AreEqual(
PTableHeader.Size + 4 * indexEntrySize + midpointsCached * indexEntrySize +
PTableFooter.GetSize(_ptableVersion) + PTable.MD5Size, fileinfo.Length);
var items = sstable.IterateAllInOrder().ToList();
Assert.AreEqual(GetHash(0x010500000000), items[0].Stream);
Assert.AreEqual(0x0001, items[0].Version);
Assert.AreEqual(GetHash(0x010200000000), items[1].Stream);
Assert.AreEqual(0x0002, items[1].Version);
Assert.AreEqual(GetHash(0x010200000000), items[2].Stream);
Assert.AreEqual(0x0001, items[2].Version);
Assert.AreEqual(GetHash(0x010100000000), items[3].Stream);
Assert.AreEqual(0x0001, items[3].Version);
}
}
private void CorruptPTableFile(string ptableFile, byte version, string corruptionType)
{
int indexEntrySize = 0;
if (version == PTableVersions.IndexV1)
{
indexEntrySize = PTable.IndexEntryV1Size;
}
else if (version == PTableVersions.IndexV2)
{
indexEntrySize = PTable.IndexEntryV2Size;
}
else if (version == PTableVersions.IndexV3)
{
indexEntrySize = PTable.IndexEntryV3Size;
}
else if (version == PTableVersions.IndexV4)
{
indexEntrySize = PTable.IndexEntryV4Size;
}
int indexEntryKeySize = 0;
if (version == PTableVersions.IndexV1)
{
indexEntryKeySize = PTable.IndexKeyV1Size;
}
else if (version == PTableVersions.IndexV2)
{
indexEntryKeySize = PTable.IndexKeyV2Size;
}
else if (version == PTableVersions.IndexV3)
{
indexEntryKeySize = PTable.IndexKeyV3Size;
}
else if (version == PTableVersions.IndexV4)
{
indexEntryKeySize = PTable.IndexKeyV4Size;
}
uint numMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries, version);
byte[] data = new byte[255];
using (FileStream stream = File.Open(ptableFile, FileMode.Open)) {
if (corruptionType == "footerFileType")
{
stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize,
SeekOrigin.Begin);
data[0] = 0xFF;
stream.Write(data, 0, 1);
}
else if (corruptionType == "footerVersion")
{
stream.Seek(
PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 1,
SeekOrigin.Begin);
data[0] = 0xFF;
stream.Write(data, 0, 1);
}
else if (corruptionType == "negativeIndexEntriesSize")
{
stream.Seek(
PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2,
SeekOrigin.Begin);
data[0] = 0xFF;
data[1] = 0xFF;
data[2] = 0xFF;
data[3] = 0xFF;
stream.Write(data, 0, 4);
}
else if (corruptionType == "notMultipleIndexEntrySize")
{
var footerPosition = PTableHeader.Size + numIndexEntries * indexEntrySize +
numMidpoints * indexEntrySize;
stream.Seek(footerPosition, SeekOrigin.Begin);
var buffer = new byte[4096];
int read = stream.Read(buffer, 0, 4096);
//insert a byte
stream.Seek(footerPosition, SeekOrigin.Begin);
data[0] = 0xFF;
stream.Write(data, 0, 1);
stream.Write(buffer, 0, read);
}
else if (corruptionType == "lessThan2Midpoints")
{
stream.Seek(
PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2,
SeekOrigin.Begin);
//1 midpoint
data[0] = 0x01;
data[1] = 0x00;
data[2] = 0x00;
data[3] = 0x00;
stream.Write(data, 0, 4);
}
else if (corruptionType == "moreMidpointsThanIndexEntries")
{
stream.Seek(
PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2,
SeekOrigin.Begin);
//change number of midpoints to 1 more than actual number to trigger the condition
var x = numMidpoints + 1;
for (int i = 0; i < 4; i++)
{
data[i] = (byte)(x & 0xFF);
x >>= 8;
}
stream.Write(data, 0, 4);
}
else if (corruptionType == "zeroOutMiddleEntries")
{
//zeroes out middle index entries - useful for binary search tests
List <int> indexEntriesToCorrupt = new List <int>();
indexEntriesToCorrupt.Add(numIndexEntries / 2 - 1);
indexEntriesToCorrupt.Add(numIndexEntries / 2);
foreach (int entry in indexEntriesToCorrupt)
{
stream.Seek(PTableHeader.Size + entry * indexEntrySize, SeekOrigin.Begin);
//modify one of the index entry hashes/version
stream.Write(data, 0, indexEntryKeySize);
if (version >= PTableVersions.IndexV4)
{
//modify one of the midpoint entry hashes/version
stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + entry * indexEntrySize,
SeekOrigin.Begin);
stream.Write(data, 0, indexEntryKeySize);
}
}
}
else if (corruptionType == "maxOutMiddleEntries")
{
//maxes out middle index entries - useful for binary search tests
List <int> indexEntriesToCorrupt = new List <int>();
indexEntriesToCorrupt.Add(numIndexEntries / 2 - 1);
indexEntriesToCorrupt.Add(numIndexEntries / 2);
for (int i = 0; i < indexEntryKeySize; i++)
{
data[i] = 0xFF;
}
foreach (int entry in indexEntriesToCorrupt)
{
stream.Seek(PTableHeader.Size + entry * indexEntrySize, SeekOrigin.Begin);
//modify one of the index entry hashes/version
stream.Write(data, 0, indexEntryKeySize);
if (version >= PTableVersions.IndexV4)
{
//modify one of the midpoint entry hashes/version
stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + entry * indexEntrySize,
SeekOrigin.Begin);
stream.Write(data, 0, indexEntryKeySize);
}
}
}
else if (corruptionType == "midpointItemIndexesNotAscendingOrder")
{
if (version >= PTableVersions.IndexV4)
{
//modify one of the midpoint item indexes
stream.Seek(
PTableHeader.Size + numIndexEntries * indexEntrySize + 3 * indexEntrySize - sizeof(long),
SeekOrigin.Begin);
stream.Write(data, 0, sizeof(long));
}
}
}
}
public void correct_number_of_midpoints_are_loaded()
{
Assert.AreEqual(_newtable.GetMidPoints().Length,
PTable.GetRequiredMidpointCountCached(14, PTableVersions.IndexV4));
}