Exemple #1
0
            private IPackedInts ReadPacked()
            {
                if (!EnsureBuffer(1 + 4))
                {
                    throw new Exception("Wrong data");
                }

                var bits  = (int)buffer[indxInBuffer];
                var count = (int)buffer[indxInBuffer + 1] | ((int)buffer[indxInBuffer + 2] << 8) | ((int)buffer[indxInBuffer + 3] << 16) | ((int)buffer[indxInBuffer + 4] << 24);
                var size  = 1 + 4 + (7 + count * bits) / 8;

                if (!EnsureBuffer(size))
                {
                    throw new Exception("Wrong data");
                }

                // NOTE: Instead of creating instance of packed integers,
                //       one can decode data from the read buffer directly.
                var packed = PackedInts.Load(buffer, indxInBuffer, size);

                indxInBuffer += size;
                return(packed);
            }
        public PostingListAddress EndList()
        {
            if (bufferIndex > 0)
            {
                buffer[deltaSelectorIndex] = deltaSelector;

                var packed = PackedInts.Convert(buffer, 0, bufferIndex).GetBytes();
                persistentStorage.WriteAll(persistentStorage.Length, packed, 0, packed.Length);

                totalSize += packed.Length;
            }

            // Write length of the list
            persistentStorage.WriteAll(listStart + sizeof(long), BitConverter.GetBytes(totalSize), 0, sizeof(int));

            var listEnd = persistentStorage.Length;

            if (listEnd - listStart != totalSize + sizeof(long) + sizeof(int))
            {
                throw new InvalidOperationException();
            }

            return(new PostingListAddress(listStart));
        }
        public void AddOccurrence(Occurrence occurrence)
        {
            if (first)
            {
                checked
                {
                    buffer[bufferIndex++] = (int)occurrence.DocumentId;
                    buffer[bufferIndex++] = (int)occurrence.FieldId;
                    buffer[bufferIndex++] = (int)occurrence.TokenId;
                }

                previous           = occurrence;
                first              = false;
                deltaSelectorIndex = bufferIndex;
                bufferIndex++;
            }
            else
            {
                int n;
                if (previous.DocumentId == occurrence.DocumentId)
                {
                    if (previous.FieldId == occurrence.FieldId)
                    {
                        n = 1;
                        checked
                        {
                            buffer[bufferIndex++] = (int)occurrence.TokenId - (int)previous.TokenId;
                        }

                        // NOTE: Removed zero value as it will lead to extra trailing occurrences
                        //       because the last deltaSelector might have unsed bits,
                        //       i.e. when deltaSelectorOffset < 32
                        // if (previous.TokenId == occurrence.TokenId)
                        // {
                        //     n = 0;
                        // }
                        // else
                        // {
                        //     n = 1;
                        //     checked
                        //     {
                        //         buffer[bufferIndex++] = (int)occurrence.TokenId - (int)previous.TokenId;
                        //     }
                        // }
                    }
                    else
                    {
                        n = 2;
                        checked
                        {
                            buffer[bufferIndex++] = (int)occurrence.FieldId - (int)previous.FieldId;
                            buffer[bufferIndex++] = (int)occurrence.TokenId;
                        }
                    }
                }
                else
                {
                    n = 3;
                    checked
                    {
                        buffer[bufferIndex++] = (int)occurrence.DocumentId - (int)previous.DocumentId;
                        buffer[bufferIndex++] = (int)occurrence.FieldId;
                        buffer[bufferIndex++] = (int)occurrence.TokenId;
                    }
                }

                previous             = occurrence;
                deltaSelector       |= (n << deltaSelectorOffset);
                deltaSelectorOffset += 2;

                // NOTE: Use less 16 bits of delta selector, so that PackedInts can compress better
                //       Might be worth experimenting with other values 4, 8 bits or even dynamic
                //       based on the contents of the buffer.
                if (deltaSelectorOffset == 16)
                {
                    buffer[deltaSelectorIndex] = deltaSelector;
                    deltaSelector       = 0;
                    deltaSelectorOffset = 0;

                    if (remainingBlocks == 1)
                    {
                        var packed = PackedInts.Convert(buffer, 0, bufferIndex).GetBytes();
                        persistentStorage.WriteAll(persistentStorage.Length, packed, 0, packed.Length);
                        totalSize += packed.Length;

                        deltaSelectorIndex = 0;
                        bufferIndex        = 1;
                        remainingBlocks    = BlocksInMemory;
                    }
                    else
                    {
                        // Reserve space for delta selector
                        deltaSelectorIndex = bufferIndex;
                        bufferIndex++;
                        remainingBlocks--;
                    }
                }
            }
        }