public void GetHistoricalLeaves_return_expected_results([ValueSource(nameof(GetHistoricalLeafTestCases))] GetHistoricalLeavesTest test)
        {
            var logFinder          = Substitute.For <ILogFinder>();
            var mainDb             = new MemDb();
            var metadaDataDb       = new MemDb();
            var baselineTreeHelper = new BaselineTreeHelper(logFinder, new MemDb(), new MemDb(), LimboNoErrorLogger.Instance);
            var baselineTree       = new ShaBaselineTree(mainDb, metadaDataDb, new byte[] { }, BaselineModule.TruncationLength, LimboNoErrorLogger.Instance);

            for (int i = 0; i < test.Blocks.Length; i++)
            {
                var block = test.Blocks[i];
                for (int j = 0; j < block.Leaves.Length; j++)
                {
                    baselineTree.Insert(block.Leaves[j]);
                }

                baselineTree.MemorizeCurrentCount(TestItem.Keccaks[block.BlockNumber], block.BlockNumber, (uint)block.Leaves.Length);
            }

            for (int i = 0; i < test.ExpectedHashes.Length; i++)
            {
                var leavesAndBlocks = test.LeavesAndBlocksQueries[i];
                var leaves          = baselineTreeHelper.GetHistoricalLeaves(baselineTree, leavesAndBlocks.LeavesIndexes, leavesAndBlocks.BlockNumber);
                for (int j = 0; j < leavesAndBlocks.LeavesIndexes.Length; j++)
                {
                    Assert.AreEqual(test.ExpectedHashes[i][j], leaves[j].Hash);
                }
            }
        }
        public void GetHistoricalTree_return_expected_results([ValueSource(nameof(HistoricalTreeTestCases))] GetHistoricalLeavesTest test)
        {
            var address            = TestItem.AddressA;
            var logFinder          = Substitute.For <ILogFinder>();
            var mainDb             = new MemDb();
            var metadataDataDb     = new MemDb();
            var baselineTreeHelper = new BaselineTreeHelper(logFinder, mainDb, metadataDataDb, LimboNoErrorLogger.Instance);
            var baselineTree       = new ShaBaselineTree(mainDb, metadataDataDb, address.Bytes, BaselineModule.TruncationLength, LimboNoErrorLogger.Instance);

            for (int i = 0; i < test.Blocks.Length; i++)
            {
                var block = test.Blocks[i];
                for (int j = 0; j < block.Leaves.Length; j++)
                {
                    baselineTree.Insert(block.Leaves[j]);
                }

                baselineTree.MemorizeCurrentCount(TestItem.Keccaks[block.BlockNumber], block.BlockNumber, (uint)block.Leaves.Length);
            }

            var historicalTree = baselineTreeHelper.CreateHistoricalTree(address, 1);

            Assert.AreNotEqual(historicalTree.Count, baselineTree.Count);
            Assert.AreNotEqual(historicalTree.Root, baselineTree.Root);
        }
Beispiel #3
0
        private BaselineTree RebuildEntireTree(Address treeAddress, Keccak blockHash)
        {
            // bad

            Keccak    leavesTopic        = new Keccak("0x8ec50f97970775682a68d3c6f9caedf60fd82448ea40706b8b65d6c03648b922");
            LogFilter insertLeavesFilter = new LogFilter(
                0,
                new BlockParameter(0L),
                new BlockParameter(blockHash),
                new AddressFilter(treeAddress),
                new TopicsFilter(new SpecificTopic(leavesTopic)));

            Keccak    leafTopic        = new Keccak("0x6a82ba2aa1d2c039c41e6e2b5a5a1090d09906f060d32af9c1ac0beff7af75c0");
            LogFilter insertLeafFilter = new LogFilter(
                0,
                new BlockParameter(0L),
                new BlockParameter(blockHash),
                new AddressFilter(treeAddress),
                new TopicsFilter(new SpecificTopic(leafTopic))); // find tree topics

            var          insertLeavesLogs = _logFinder.FindLogs(insertLeavesFilter);
            var          insertLeafLogs   = _logFinder.FindLogs(insertLeafFilter);
            BaselineTree baselineTree     = new ShaBaselineTree(new MemDb(), Array.Empty <byte>(), 5);

            // Keccak leafTopic = new Keccak("0x8ec50f97970775682a68d3c6f9caedf60fd82448ea40706b8b65d6c03648b922");
            foreach (FilterLog filterLog in insertLeavesLogs
                     .Union(insertLeafLogs)
                     .OrderBy(fl => fl.BlockNumber).ThenBy(fl => fl.LogIndex))
            {
                if (filterLog.Data.Length == 96)
                {
                    Keccak leafHash = new Keccak(filterLog.Data.Slice(32, 32).ToArray());
                    baselineTree.Insert(leafHash);
                }
                else
                {
                    for (int i = 0; i < (filterLog.Data.Length - 128) / 32; i++)
                    {
                        Keccak leafHash = new Keccak(filterLog.Data.Slice(128 + 32 * i, 32).ToArray());
                        baselineTree.Insert(leafHash);
                    }
                }
            }

            return(baselineTree);
        }
Beispiel #4
0
        public void InsertingValuesWithoutCalculatingHashes()
        {
            BaselineTree baselineTree = new ShaBaselineTree(new MemDb(), new MemDb(), Array.Empty <byte>(), 0, LimboNoErrorLogger.Instance);

            for (uint i = 0; i < _testLeaves.Length; ++i)
            {
                baselineTree.Insert(_testLeaves[i], false);
            }
        }
Beispiel #5
0
        public void BuildTreeWithInstantHashing()
        {
            BaselineTree baselineTree = new ShaBaselineTree(new MemDb(), new MemDb(), Array.Empty <byte>(), 0, LimboNoErrorLogger.Instance);

            for (uint i = 0; i < _testLeaves.Length; ++i)
            {
                baselineTree.Insert(_testLeaves[i]);
            }
        }
Beispiel #6
0
        private BaselineTree RebuildEntireTree(Address treeAddress)
        {
            // bad

            Keccak    leavesTopic        = new Keccak("0x8ec50f97970775682a68d3c6f9caedf60fd82448ea40706b8b65d6c03648b922");
            LogFilter insertLeavesFilter = new LogFilter(
                0,
                new BlockParameter(0L),
                new BlockParameter(_blockFinder.Head.Number),
                new AddressFilter(treeAddress),
                new TopicsFilter(new SpecificTopic(leavesTopic)));

            Keccak    leafTopic        = new Keccak("0x6a82ba2aa1d2c039c41e6e2b5a5a1090d09906f060d32af9c1ac0beff7af75c0");
            LogFilter insertLeafFilter = new LogFilter(
                0,
                new BlockParameter(0L),
                new BlockParameter(_blockFinder.Head.Number),
                new AddressFilter(treeAddress),
                new TopicsFilter(new SpecificTopic(leafTopic))); // find tree topics

            var          insertLeavesLogs = _logFinder.FindLogs(insertLeavesFilter);
            var          insertLeafLogs   = _logFinder.FindLogs(insertLeafFilter);
            BaselineTree baselineTree     = new ShaBaselineTree(new MemDb(), new byte[0], 5);

            // Keccak leafTopic = new Keccak("0x8ec50f97970775682a68d3c6f9caedf60fd82448ea40706b8b65d6c03648b922");
            foreach (FilterLog filterLog in insertLeavesLogs)
            {
                for (int i = 0; i < (filterLog.Data.Length - 128) / 32; i++)
                {
                    Bytes32 leafHash = Bytes32.Wrap(filterLog.Data.Slice(128 + 32 * i, 32).ToArray());
                    baselineTree.Insert(leafHash);
                }
            }

            foreach (FilterLog filterLog in insertLeafLogs)
            {
                Bytes32 leafHash = Bytes32.Wrap(filterLog.Data.Slice(32, 32).ToArray());
                baselineTree.Insert(leafHash);
            }

            return(baselineTree);
        }
Beispiel #7
0
        public void BuildTreeWithHashingInTheEnd()
        {
            BaselineTree baselineTree = new ShaBaselineTree(new MemDb(), new MemDb(), Array.Empty <byte>(), 0, LimboNoErrorLogger.Instance);

            for (uint i = 0; i < _testLeaves.Length; ++i)
            {
                baselineTree.Insert(_testLeaves[i], false);
            }

            baselineTree.CalculateHashes();
        }
        public void Can_restore_count_from_the_database(uint leafCount)
        {
            MemDb        memDb         = new MemDb();
            var          metadataMemDb = new MemDb();
            BaselineTree baselineTree  = new ShaBaselineTree(memDb, metadataMemDb, new byte[] { }, _truncationLength, LimboNoErrorLogger.Instance);

            for (int i = 0; i < leafCount; i++)
            {
                baselineTree.Insert(_testLeaves[0]);
            }

            BaselineTree baselineTreeRestored = new ShaBaselineTree(memDb, metadataMemDb, new byte[] { }, _truncationLength, LimboNoErrorLogger.Instance);

            baselineTreeRestored.Count.Should().Be(leafCount);
        }
        // TODO: fuzzer with concurrent inserts
        public void Baseline_tree_fuzzer(
            int leavesPerBlock,
            int blocksCount,
            int emptyBlocksRatio,
            bool recalculateOnInsert,
            bool withReorgs,
            int?randomSeed)
        {
            MemDb              mainDb     = new MemDb();
            MemDb              metadataDb = new MemDb();
            Address            address    = Address.Zero;
            BaselineTreeHelper helper     = new BaselineTreeHelper(
                Substitute.For <ILogFinder>(), mainDb, metadataDb, LimboNoErrorLogger.Instance);
            BaselineTree baselineTree = new ShaBaselineTree(
                mainDb, metadataDb, address.Bytes, 0, LimboNoErrorLogger.Instance);

            randomSeed ??= _random.Next();
            Console.WriteLine($"random seed was {randomSeed} - hardcode it to recreate the failign test");
            // Random random = new Random(1524199427); <- example
            Random                  random                   = new Random(randomSeed.Value);
            int                     currentBlockNumber       = 0;
            uint                    totalCountCheck          = 0;
            Stack <long>            lastBlockWithLeavesCheck = new Stack <long>();
            Dictionary <long, uint> historicalCountChecks    = new Dictionary <long, uint>();

            historicalCountChecks[0] = 0;
            for (int i = 0; i < blocksCount; i++)
            {
                if (i == 18)
                {
                }
                currentBlockNumber++;
                uint numberOfLeaves = (uint)random.Next(leavesPerBlock) + 1;  // not zero
                bool hasLeaves      = random.Next(100) < emptyBlocksRatio;

                if (hasLeaves)
                {
                    totalCountCheck += numberOfLeaves;

                    TestContext.WriteLine($"Adding {numberOfLeaves} at block {currentBlockNumber}");
                    for (int j = 0; j < numberOfLeaves; j++)
                    {
                        byte[] leafBytes = new byte[32];
                        random.NextBytes(leafBytes);
                        baselineTree.Insert(new Keccak(leafBytes), recalculateOnInsert);
                    }

                    lastBlockWithLeavesCheck.TryPeek(out long previous);
                    TestContext.WriteLine($"Previous is {previous}");
                    baselineTree.LastBlockWithLeaves.Should().Be(previous);
                    baselineTree.MemorizeCurrentCount(TestItem.Keccaks[currentBlockNumber], currentBlockNumber, baselineTree.Count);
                    lastBlockWithLeavesCheck.Push(currentBlockNumber);

                    baselineTree.Count.Should().Be(totalCountCheck);
                    baselineTree.LastBlockWithLeaves.Should().Be(lastBlockWithLeavesCheck.Peek());
                }
                else
                {
                    TestContext.WriteLine($"Block {currentBlockNumber} has no leaves");
                }

                historicalCountChecks[currentBlockNumber] = totalCountCheck;

                WriteHistory(historicalCountChecks, baselineTree);

                for (int j = 1; j <= currentBlockNumber; j++)
                {
                    TestContext.WriteLine($"Creating historical at {j}");
                    var historicalTrie = helper.CreateHistoricalTree(address, j);
                    TestContext.WriteLine($"Checking if trie count ({historicalTrie.Count}) is {historicalCountChecks[j]} as expected");
                    historicalTrie.Count.Should().Be(historicalCountChecks[j], $"Block is {currentBlockNumber}, checking count at block {j}.");
                }

                if (withReorgs)
                {
                    bool shouldReorg = random.Next(100) < 50;
                    if (shouldReorg && currentBlockNumber >= 1)
                    {
                        int reorgDepth = random.Next(currentBlockNumber) + 1;
                        TestContext.WriteLine($"Reorganizing {reorgDepth} from {currentBlockNumber}");
                        uint expectedDeleteCount = historicalCountChecks[currentBlockNumber] - historicalCountChecks[currentBlockNumber - reorgDepth];
                        baselineTree.GoBackTo(currentBlockNumber - reorgDepth).Should().Be(expectedDeleteCount);
                        for (int j = 0; j < reorgDepth; j++)
                        {
                            historicalCountChecks.Remove(currentBlockNumber - j);
                        }

                        currentBlockNumber -= reorgDepth;
                        totalCountCheck     = historicalCountChecks[currentBlockNumber];
                        baselineTree.MemorizeCurrentCount(TestItem.Keccaks[currentBlockNumber], currentBlockNumber, totalCountCheck);

                        TestContext.WriteLine($"Total count after reorg is {totalCountCheck} at block {currentBlockNumber}");


                        while (lastBlockWithLeavesCheck.Any() && lastBlockWithLeavesCheck.Peek() > currentBlockNumber)
                        {
                            lastBlockWithLeavesCheck.Pop();
                        }

                        lastBlockWithLeavesCheck.TryPeek(out long last);
                        if (last != currentBlockNumber)
                        {
                            TestContext.WriteLine($"Pushing {currentBlockNumber} on test stack after reorg.");
                            // after reorg we always push a memorized count
                            lastBlockWithLeavesCheck.Push(currentBlockNumber);
                        }
                    }

                    WriteHistory(historicalCountChecks, baselineTree);
                }
            }
        }