Пример #1
0
        public void file_size_should_be_greater_or_equal_after_adding_midpoints(byte version, int minDepth)
        {
            for (long numIndexEntries = 0; numIndexEntries < MAX_INDEX_ENTRIES; numIndexEntries += getIncrement(numIndexEntries))
            {
                string testCase      = $"numIndexEntries: {numIndexEntries}, minDepth: {minDepth}, version: {version}";
                var    midpointCount = PTable.GetRequiredMidpointCountCached(numIndexEntries, version, minDepth);
                if (version < PTableVersions.IndexV4)
                {
                    Assert.AreEqual(0, midpointCount, testCase);
                }
                else
                {
                    Assert.GreaterOrEqual(midpointCount, Math.Min(1 << Math.Min(MAX_DEPTH, minDepth), numIndexEntries), testCase);
                    Assert.LessOrEqual(midpointCount, Math.Max(2, numIndexEntries), testCase);
                }

                long fileSizeUpToIndexEntries    = PTable.GetFileSizeUpToIndexEntries(numIndexEntries, version);
                long fileSizeUpToMidpointEntries =
                    PTable.GetFileSizeUpToMidpointEntries(fileSizeUpToIndexEntries, midpointCount, version);

                if (version < PTableVersions.IndexV4)
                {
                    Assert.AreEqual(fileSizeUpToIndexEntries, fileSizeUpToMidpointEntries, testCase);
                }
                else
                {
                    Assert.GreaterOrEqual(fileSizeUpToMidpointEntries, fileSizeUpToIndexEntries, testCase);
                }
            }
        }
        private void construct_same_midpoint_indexes_for_any_combination_of_params(int maxIndexEntries)
        {
            for (var numIndexEntries = 0; numIndexEntries < maxIndexEntries; numIndexEntries++)
            {
                for (var depth = 0; depth < 20; depth++)
                {
                    var requiredMidpointsCount =
                        PTable.GetRequiredMidpointCountCached(numIndexEntries, _ptableVersion, depth);
                    List <long> requiredMidpoints = new List <long>();
                    for (var k = 0; k < requiredMidpointsCount; k++)
                    {
                        var index = PTable.GetMidpointIndex(k, numIndexEntries, requiredMidpointsCount);
                        requiredMidpoints.Add(index);
                    }

                    List <long> calculatedMidpoints = new List <long>();
                    for (var k = 0; k < numIndexEntries; k++)
                    {
                        if (PTable.IsMidpointIndex(k, numIndexEntries, requiredMidpointsCount))
                        {
                            calculatedMidpoints.Add(k);
                        }
                    }

                    if (numIndexEntries == 1 && calculatedMidpoints.Count == 1)
                    {
                        calculatedMidpoints.Add(calculatedMidpoints[0]);
                    }

                    if (requiredMidpoints.Count != calculatedMidpoints.Count)
                    {
                        Log.Error(
                            "Midpoint count mismatch for numIndexEntries: {0}, depth:{1} - Expected {2}, Found {3}",
                            numIndexEntries, depth, requiredMidpoints.Count, calculatedMidpoints.Count);
                    }

                    Assert.AreEqual(requiredMidpoints.Count, calculatedMidpoints.Count);

                    for (var i = 0; i < requiredMidpoints.Count; i++)
                    {
                        if (requiredMidpoints[i] != calculatedMidpoints[i])
                        {
                            Log.Error(
                                "Midpoint mismatch at index {0} for numIndexEntries: {1}, depth:{2} - Expected {3}, Found {4}",
                                i, numIndexEntries, depth, requiredMidpoints[i], calculatedMidpoints[i]);
                        }

                        Assert.AreEqual(requiredMidpoints[i], calculatedMidpoints[i]);
                    }
                }
            }
        }
        public void the_file_gets_created_and_can_be_read()
        {
            var indexEntrySize = PTable.IndexEntryV4Size;

            if (_ptableVersion == PTableVersions.IndexV1)
            {
                indexEntrySize = PTable.IndexEntryV1Size;
            }
            else if (_ptableVersion == PTableVersions.IndexV2)
            {
                indexEntrySize = PTable.IndexEntryV2Size;
            }
            else if (_ptableVersion == PTableVersions.IndexV3)
            {
                indexEntrySize = PTable.IndexEntryV3Size;
            }

            var table = new HashListMemTable(_ptableVersion, maxSize: 10);

            table.Add(0x010100000000, 0x0001, 0x0001);
            table.Add(0x010500000000, 0x0001, 0x0002);
            table.Add(0x010200000000, 0x0001, 0x0003);
            table.Add(0x010200000000, 0x0002, 0x0003);
            using (var sstable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault,
                                                     skipIndexVerify: _skipIndexVerify,
                                                     useBloomFilter: true)) {
                var fileinfo        = new FileInfo(Filename);
                var midpointsCached = PTable.GetRequiredMidpointCountCached(4, _ptableVersion);
                Assert.AreEqual(
                    PTableHeader.Size + 4 * indexEntrySize + midpointsCached * indexEntrySize +
                    PTableFooter.GetSize(_ptableVersion) + PTable.MD5Size, fileinfo.Length);
                var items = sstable.IterateAllInOrder().ToList();
                Assert.AreEqual(GetHash(0x010500000000), items[0].Stream);
                Assert.AreEqual(0x0001, items[0].Version);
                Assert.AreEqual(GetHash(0x010200000000), items[1].Stream);
                Assert.AreEqual(0x0002, items[1].Version);
                Assert.AreEqual(GetHash(0x010200000000), items[2].Stream);
                Assert.AreEqual(0x0001, items[2].Version);
                Assert.AreEqual(GetHash(0x010100000000), items[3].Stream);
                Assert.AreEqual(0x0001, items[3].Version);

                var stream = (ulong)0x010200000000;
                Assert.True(sstable.TryGetLatestEntry(stream, out var entry));
                Assert.AreEqual(GetHash(stream), entry.Stream);
                Assert.AreEqual(0x0002, entry.Version);
                Assert.AreEqual(0x0003, entry.Position);
            }
        }
Пример #4
0
 public void for_a_fixed_number_of_index_entries_and_increasing_min_depth_midpoint_count_should_increase_monotonically()
 {
     for (long numIndexEntries = 0; numIndexEntries < MAX_INDEX_ENTRIES; numIndexEntries += getIncrement(numIndexEntries))
     {
         long prevMidpointCount = 0L;
         for (int minDepth = 0; minDepth <= MAX_TEST_DEPTH; minDepth++)
         {
             string testCase      = $"numIndexEntries: {numIndexEntries}, minDepth: {minDepth}";
             var    midpointCount = PTable.GetRequiredMidpointCountCached(numIndexEntries, PTableVersions.IndexV4, minDepth);
             Assert.GreaterOrEqual(midpointCount, prevMidpointCount, testCase);
             Assert.GreaterOrEqual(midpointCount, Math.Min(1 << Math.Min(MAX_DEPTH, minDepth), numIndexEntries), testCase);
             Assert.LessOrEqual(midpointCount, Math.Max(2, numIndexEntries), testCase);
             prevMidpointCount = midpointCount;
         }
     }
 }
        public void the_correct_midpoints_are_cached()
        {
            PTable.Midpoint[] midpoints = _newtable.GetMidPoints();
            var requiredMidpoints       = PTable.GetRequiredMidpointCountCached(_newtable.Count, _ptableVersion);

            Assert.AreEqual(requiredMidpoints, midpoints.Length);

            var position = 0;

            foreach (var item in _newtable.IterateAllInOrder())
            {
                if (PTable.IsMidpointIndex(position, _newtable.Count, requiredMidpoints))
                {
                    Assert.AreEqual(item.Stream, midpoints[position].Key.Stream);
                    Assert.AreEqual(item.Version, midpoints[position].Key.Version);
                    Assert.AreEqual(position, midpoints[position].ItemIndex);
                    position++;
                }
            }
        }
Пример #6
0
        public void midpoints_are_cached_in_ptable_footer()
        {
            var numIndexEntries   = 14;
            var requiredMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries, PTableVersions.IndexV4);

            var newTableFileCopy = GetTempFilePath();

            File.Copy(_newtableFile, newTableFileCopy);
            using (var filestream = File.Open(newTableFileCopy, FileMode.Open, FileAccess.Read))
            {
                var footerSize = PTableFooter.GetSize(PTableVersions.IndexV4);
                Assert.AreEqual(filestream.Length, PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size + footerSize + PTable.MD5Size);
                filestream.Seek(PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size, SeekOrigin.Begin);

                var ptableFooter = PTableFooter.FromStream(filestream);
                Assert.AreEqual(FileType.PTableFile, ptableFooter.FileType);
                Assert.AreEqual(PTableVersions.IndexV4, ptableFooter.Version);
                Assert.AreEqual(requiredMidpoints, ptableFooter.NumMidpointsCached);
            }
        }
Пример #7
0
        public void the_file_gets_created()
        {
            var indexEntrySize = PTable.IndexEntryV4Size;

            if (_ptableVersion == PTableVersions.IndexV1)
            {
                indexEntrySize = PTable.IndexEntryV1Size;
            }
            else if (_ptableVersion == PTableVersions.IndexV2)
            {
                indexEntrySize = PTable.IndexEntryV2Size;
            }
            else if (_ptableVersion == PTableVersions.IndexV3)
            {
                indexEntrySize = PTable.IndexEntryV3Size;
            }

            var table = new HashListMemTable(_ptableVersion, maxSize: 10);

            table.Add(0x010100000000, 0x0001, 0x0001);
            table.Add(0x010500000000, 0x0001, 0x0002);
            table.Add(0x010200000000, 0x0001, 0x0003);
            table.Add(0x010200000000, 0x0002, 0x0003);
            using (var sstable = PTable.FromMemtable(table, Filename, skipIndexVerify: _skipIndexVerify)) {
                var fileinfo        = new FileInfo(Filename);
                var midpointsCached = PTable.GetRequiredMidpointCountCached(4, _ptableVersion);
                Assert.AreEqual(
                    PTableHeader.Size + 4 * indexEntrySize + midpointsCached * indexEntrySize +
                    PTableFooter.GetSize(_ptableVersion) + PTable.MD5Size, fileinfo.Length);
                var items = sstable.IterateAllInOrder().ToList();
                Assert.AreEqual(GetHash(0x010500000000), items[0].Stream);
                Assert.AreEqual(0x0001, items[0].Version);
                Assert.AreEqual(GetHash(0x010200000000), items[1].Stream);
                Assert.AreEqual(0x0002, items[1].Version);
                Assert.AreEqual(GetHash(0x010200000000), items[2].Stream);
                Assert.AreEqual(0x0001, items[2].Version);
                Assert.AreEqual(GetHash(0x010100000000), items[3].Stream);
                Assert.AreEqual(0x0001, items[3].Version);
            }
        }
Пример #8
0
        private void CorruptPTableFile(string ptableFile, byte version, string corruptionType)
        {
            int indexEntrySize = 0;

            if (version == PTableVersions.IndexV1)
            {
                indexEntrySize = PTable.IndexEntryV1Size;
            }
            else if (version == PTableVersions.IndexV2)
            {
                indexEntrySize = PTable.IndexEntryV2Size;
            }
            else if (version == PTableVersions.IndexV3)
            {
                indexEntrySize = PTable.IndexEntryV3Size;
            }
            else if (version == PTableVersions.IndexV4)
            {
                indexEntrySize = PTable.IndexEntryV4Size;
            }

            int indexEntryKeySize = 0;

            if (version == PTableVersions.IndexV1)
            {
                indexEntryKeySize = PTable.IndexKeyV1Size;
            }
            else if (version == PTableVersions.IndexV2)
            {
                indexEntryKeySize = PTable.IndexKeyV2Size;
            }
            else if (version == PTableVersions.IndexV3)
            {
                indexEntryKeySize = PTable.IndexKeyV3Size;
            }
            else if (version == PTableVersions.IndexV4)
            {
                indexEntryKeySize = PTable.IndexKeyV4Size;
            }

            uint numMidpoints = PTable.GetRequiredMidpointCountCached(numIndexEntries, version);

            byte[] data = new byte[255];

            using (FileStream stream = File.Open(ptableFile, FileMode.Open)) {
                if (corruptionType == "footerFileType")
                {
                    stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize,
                                SeekOrigin.Begin);
                    data[0] = 0xFF;
                    stream.Write(data, 0, 1);
                }
                else if (corruptionType == "footerVersion")
                {
                    stream.Seek(
                        PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 1,
                        SeekOrigin.Begin);
                    data[0] = 0xFF;
                    stream.Write(data, 0, 1);
                }
                else if (corruptionType == "negativeIndexEntriesSize")
                {
                    stream.Seek(
                        PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2,
                        SeekOrigin.Begin);
                    data[0] = 0xFF;
                    data[1] = 0xFF;
                    data[2] = 0xFF;
                    data[3] = 0xFF;
                    stream.Write(data, 0, 4);
                }
                else if (corruptionType == "notMultipleIndexEntrySize")
                {
                    var footerPosition = PTableHeader.Size + numIndexEntries * indexEntrySize +
                                         numMidpoints * indexEntrySize;
                    stream.Seek(footerPosition, SeekOrigin.Begin);
                    var buffer = new byte[4096];
                    int read   = stream.Read(buffer, 0, 4096);

                    //insert a byte
                    stream.Seek(footerPosition, SeekOrigin.Begin);
                    data[0] = 0xFF;
                    stream.Write(data, 0, 1);
                    stream.Write(buffer, 0, read);
                }
                else if (corruptionType == "lessThan2Midpoints")
                {
                    stream.Seek(
                        PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2,
                        SeekOrigin.Begin);
                    //1 midpoint
                    data[0] = 0x01;
                    data[1] = 0x00;
                    data[2] = 0x00;
                    data[3] = 0x00;
                    stream.Write(data, 0, 4);
                }
                else if (corruptionType == "moreMidpointsThanIndexEntries")
                {
                    stream.Seek(
                        PTableHeader.Size + numIndexEntries * indexEntrySize + numMidpoints * indexEntrySize + 2,
                        SeekOrigin.Begin);
                    //change number of midpoints to 1 more than actual number to trigger the condition
                    var x = numMidpoints + 1;
                    for (int i = 0; i < 4; i++)
                    {
                        data[i] = (byte)(x & 0xFF);
                        x     >>= 8;
                    }

                    stream.Write(data, 0, 4);
                }
                else if (corruptionType == "zeroOutMiddleEntries")
                {
                    //zeroes out middle index entries - useful for binary search tests
                    List <int> indexEntriesToCorrupt = new List <int>();
                    indexEntriesToCorrupt.Add(numIndexEntries / 2 - 1);
                    indexEntriesToCorrupt.Add(numIndexEntries / 2);

                    foreach (int entry in indexEntriesToCorrupt)
                    {
                        stream.Seek(PTableHeader.Size + entry * indexEntrySize, SeekOrigin.Begin);
                        //modify one of the index entry hashes/version
                        stream.Write(data, 0, indexEntryKeySize);

                        if (version >= PTableVersions.IndexV4)
                        {
                            //modify one of the midpoint entry hashes/version
                            stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + entry * indexEntrySize,
                                        SeekOrigin.Begin);
                            stream.Write(data, 0, indexEntryKeySize);
                        }
                    }
                }
                else if (corruptionType == "maxOutMiddleEntries")
                {
                    //maxes out middle index entries - useful for binary search tests
                    List <int> indexEntriesToCorrupt = new List <int>();
                    indexEntriesToCorrupt.Add(numIndexEntries / 2 - 1);
                    indexEntriesToCorrupt.Add(numIndexEntries / 2);

                    for (int i = 0; i < indexEntryKeySize; i++)
                    {
                        data[i] = 0xFF;
                    }

                    foreach (int entry in indexEntriesToCorrupt)
                    {
                        stream.Seek(PTableHeader.Size + entry * indexEntrySize, SeekOrigin.Begin);
                        //modify one of the index entry hashes/version
                        stream.Write(data, 0, indexEntryKeySize);

                        if (version >= PTableVersions.IndexV4)
                        {
                            //modify one of the midpoint entry hashes/version
                            stream.Seek(PTableHeader.Size + numIndexEntries * indexEntrySize + entry * indexEntrySize,
                                        SeekOrigin.Begin);
                            stream.Write(data, 0, indexEntryKeySize);
                        }
                    }
                }
                else if (corruptionType == "midpointItemIndexesNotAscendingOrder")
                {
                    if (version >= PTableVersions.IndexV4)
                    {
                        //modify one of the midpoint item indexes
                        stream.Seek(
                            PTableHeader.Size + numIndexEntries * indexEntrySize + 3 * indexEntrySize - sizeof(long),
                            SeekOrigin.Begin);
                        stream.Write(data, 0, sizeof(long));
                    }
                }
            }
        }
Пример #9
0
 public void correct_number_of_midpoints_are_loaded()
 {
     Assert.AreEqual(_newtable.GetMidPoints().Length,
                     PTable.GetRequiredMidpointCountCached(14, PTableVersions.IndexV4));
 }