Exemple #1
0
        /// <summary>
        /// Merge some number of blocks, how many is decided by mergeFactor, into a single sorted block. This is done by opening <seealso cref="BlockEntryReader"/> on each
        /// block that we want to merge and give them to a <seealso cref="MergingBlockEntryReader"/>. The <seealso cref="BlockEntryReader"/>s are pulled from a <seealso cref="BlockReader"/>
        /// that iterate over Blocks in file in sequential order.
        ///
        /// <seealso cref="MergingBlockEntryReader"/> pull head from each <seealso cref="BlockEntryReader"/> and hand them out in sorted order, making the multiple entry readers look
        /// like a single large and sorted entry reader.
        ///
        /// The large block resulting from the merge is written down to targetChannel by calling
        /// <seealso cref="writeBlock(StoreChannel, BlockEntryCursor, long, long, Cancellation, IntConsumer, ByteBuffer)"/>.
        /// </summary>
        /// <param name="mergeFactor"> How many blocks to merge at the same time. Influence how much memory will be used because each merge block will have it's own
        /// <seealso cref="ByteBuffer"/> that they read from. </param>
        /// <param name="reader"> The <seealso cref="BlockReader"/> to pull blocks / <seealso cref="BlockEntryReader"/>s from. </param>
        /// <param name="targetChannel"> The <seealso cref="StoreChannel"/> to write the merge result to. Result will be appended to current position. </param>
        /// <param name="cancellation"> Injected so that this merge can be cancelled, if an external request to do that comes in. </param>
        /// <param name="readBuffers"> buffers for all block readers. </param>
        /// <param name="writeBuffer"> buffer for writing merged blocks. </param>
        /// <returns> The number of blocks that where merged, most often this will be equal to mergeFactor but can be less if there are fewer blocks left in source. </returns>
        /// <exception cref="IOException"> If something goes wrong when reading from file. </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private int performSingleMerge(int mergeFactor, BlockReader<KEY,VALUE> reader, org.neo4j.io.fs.StoreChannel targetChannel, Cancellation cancellation, ByteBuffer[] readBuffers, ByteBuffer writeBuffer) throws java.io.IOException
        private int PerformSingleMerge(int mergeFactor, BlockReader <KEY, VALUE> reader, StoreChannel targetChannel, Cancellation cancellation, ByteBuffer[] readBuffers, ByteBuffer writeBuffer)
        {
            using (MergingBlockEntryReader <KEY, VALUE> merger = new MergingBlockEntryReader <KEY, VALUE>(_layout))
            {
                long blockSize    = 0;
                long entryCount   = 0;
                int  blocksMerged = 0;
                for (int i = 0; i < mergeFactor; i++)
                {
                    readBuffers[i].clear();
                    BlockEntryReader <KEY, VALUE> source = reader.NextBlock(readBuffers[i]);
                    if (source != null)
                    {
                        blockSize  += source.BlockSize();
                        entryCount += source.EntryCount();
                        blocksMerged++;
                        merger.AddSource(source);
                    }
                    else
                    {
                        break;
                    }
                }

                writeBuffer.clear();
                WriteBlock(targetChannel, merger, blockSize, entryCount, cancellation, _monitor.entriesMerged, writeBuffer);
                _monitor.mergedBlocks(blockSize, entryCount, blocksMerged);
                return(blocksMerged);
            }
        }
Exemple #2
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void writeScanUpdatesToTree(RecordingConflictDetector<KEY,VALUE> recordingConflictDetector, org.neo4j.kernel.impl.index.schema.ByteBufferFactory.Allocator allocator, int bufferSize) throws java.io.IOException, org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException
        private void WriteScanUpdatesToTree(RecordingConflictDetector <KEY, VALUE> recordingConflictDetector, Allocator allocator, int bufferSize)
        {
            using (MergingBlockEntryReader <KEY, VALUE> allEntries = new MergingBlockEntryReader <KEY, VALUE>(layout))
            {
                ByteBuffer singleBlockAssertionBuffer = allocator.Allocate(( int )kibiBytes(8));
                foreach (ThreadLocalBlockStorage part in _allScanUpdates)
                {
                    using (BlockReader <KEY, VALUE> reader = part.BlockStorage.reader())
                    {
                        BlockEntryReader <KEY, VALUE> singleMergedBlock = reader.NextBlock(allocator.Allocate(bufferSize));
                        if (singleMergedBlock != null)
                        {
                            allEntries.AddSource(singleMergedBlock);
                            // Pass in some sort of ByteBuffer here. The point is that there should be no more data to read,
                            // if there is then it's due to a bug in the code and must be fixed.
                            if (reader.NextBlock(singleBlockAssertionBuffer) != null)
                            {
                                throw new System.InvalidOperationException("Final BlockStorage had multiple blocks");
                            }
                        }
                    }
                }

                int asMuchAsPossibleToTheLeft = 1;
                using (Writer <KEY, VALUE> writer = tree.writer(asMuchAsPossibleToTheLeft))
                {
                    while (allEntries.Next() && !_cancellation.cancelled())
                    {
                        WriteToTree(writer, recordingConflictDetector, allEntries.Key(), allEntries.Value());
                        _numberOfAppliedScanUpdates++;
                    }
                }
            }
        }
Exemple #3
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: BlockEntryReader<KEY,VALUE> nextBlock(ByteBuffer blockBuffer) throws java.io.IOException
        internal virtual BlockEntryReader <KEY, VALUE> NextBlock(ByteBuffer blockBuffer)
        {
            long position = _channel.position();

            if (position >= _channel.size())
            {
                return(null);
            }
            StoreChannel blockChannel = _fs.open(_file, OpenMode.READ);

            blockChannel.Position(position);
            PageCursor pageCursor = new ReadableChannelPageCursor(new ReadAheadChannel <>(blockChannel, blockBuffer));
            BlockEntryReader <KEY, VALUE> blockEntryReader = new BlockEntryReader <KEY, VALUE>(pageCursor, _layout);
            long blockSize = blockEntryReader.BlockSize();

            _channel.position(position + blockSize);
            return(blockEntryReader);
        }
Exemple #4
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void assertContents(org.neo4j.index.internal.gbptree.SimpleLongLayout layout, BlockStorage<org.apache.commons.lang3.mutable.MutableLong,org.apache.commons.lang3.mutable.MutableLong> storage, Iterable<java.util.List<BlockEntry<org.apache.commons.lang3.mutable.MutableLong,org.apache.commons.lang3.mutable.MutableLong>>> expectedBlocks) throws java.io.IOException
        private void AssertContents(SimpleLongLayout layout, BlockStorage <MutableLong, MutableLong> storage, IEnumerable <IList <BlockEntry <MutableLong, MutableLong> > > expectedBlocks)
        {
            using (BlockReader <MutableLong, MutableLong> reader = storage.Reader())
            {
                foreach (IList <BlockEntry <MutableLong, MutableLong> > expectedBlock in expectedBlocks)
                {
                    using (BlockEntryReader <MutableLong, MutableLong> block = reader.NextBlock(HEAP_ALLOCATOR.allocate(1024)))
                    {
                        assertNotNull(block);
                        assertEquals(expectedBlock.Count, block.EntryCount());
                        foreach (BlockEntry <MutableLong, MutableLong> expectedEntry in expectedBlock)
                        {
                            assertTrue(block.Next());
                            assertEquals(0, layout.Compare(expectedEntry.Key(), block.Key()));
                            assertEquals(expectedEntry.Value(), block.Value());
                        }
                    }
                }
            }
        }