/// <summary>
/// Searches the index for items
/// </summary>
/// <param name="tokens">The list of tokens which have to occur in the document in the given order</param>
/// <returns>An enumerable returning the matched items</returns>
public IEnumerable <IndexItem> Search(Token[] tokens)
{
// return all lines if no tokens given
if (tokens.Length == 0)
{
var files = new List <Tuple <string, string, IReadOnlyList <uint>, IReadOnlyList <uint> > >();
var fileStart = 0;
foreach (var file in _lines.OrderBy(i => i.File, _comparer).ThenBy(i => i.Member, _comparer))
{
var lines = new EliasFanoList((uint)(fileStart + file.Positions.Count), file.Positions.Count, Enumerable.Range(fileStart, file.Positions.Count).Select(i => (uint)i));
files.Add(new Tuple <string, string, IReadOnlyList <uint>, IReadOnlyList <uint> >(file.File, file.Member, file.Positions, lines));
fileStart += file.Positions.Count;
}
// return the original compressed index in case we want to compress it later on (highly possible when returning all lines -> full view of log is visible)
return(new DirectIndex(files));
}
else
{
return(SearchIterator(tokens));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="EliasFanoEnumerator"/>
/// </summary>
/// <param name="collection">List to enumerate on</param>
public EliasFanoEnumerator(EliasFanoList collection)
{
_collection = collection;
_lowerBitsMask = (uint)(1 << _collection._lowerBitCount) - 1;
Reset();
}
/// <summary>
/// Adds an item to the list
/// </summary>
/// <param name="item">Item to add</param>
public void Add(uint item)
{
if (item < _lastAddedItem)
{
throw new InvalidOperationException("Items must be added in monotonically increasing order");
}
if (item > _universe)
{
// extend the universe and recompress the list
var temp = new EliasFanoList(_universe * 2, Count + 1, this.Concat(new[] { item }));
_lowerBitCount = temp._lowerBitCount;
_universe = temp._universe;
_skipQuantum = temp._skipQuantum;
_indexQuantum = temp._indexQuantum;
_count = temp._count;
_upperBitCount = temp._upperBitCount;
_lowerBits = temp._lowerBits;
_upperBits = temp._upperBits;
_skipPointers = temp._skipPointers;
_skipPointerCount = temp._skipPointerCount;
_indexPointers = temp._indexPointers;
_indexPointerCount = temp._indexPointerCount;
_lastAddedItem = temp._lastAddedItem;
_negatedUnaryReadCounter = temp._negatedUnaryReadCounter;
_unaryReadCounter = temp._unaryReadCounter;
_indexEnumerator.Reset();
}
else
{
// store the lower bits
var lowerBitMask = (1 << _lowerBitCount) - 1;
var lowerBits = item & lowerBitMask;
var lowerBitCounter = 0;
var lowerBitPointer = _lowerBitCount * _count;
while (lowerBitCounter < _lowerBitCount)
{
var byteOffset = (lowerBitPointer + lowerBitCounter) / 32;
var bitOffset = (lowerBitPointer + lowerBitCounter) % 32;
// resize the array if required
if (byteOffset >= _lowerBits.Length)
{
Array.Resize(ref _lowerBits, (int)Math.Ceiling(byteOffset * 1.5));
}
_lowerBits[byteOffset] |= (uint)(lowerBits << bitOffset);
lowerBitCounter += 32 - bitOffset;
lowerBits >>= 32 - bitOffset;
}
lowerBitPointer += _lowerBitCount;
// get the upper bits to store and the difference to the previous upper bits
var upperBits = item >> _lowerBitCount;
var delta = upperBits - (_lastAddedItem >> _lowerBitCount);
// build the negated unary skip list
for (var i = 0U; i < delta; i++)
{
_negatedUnaryReadCounter += 1;
if (_negatedUnaryReadCounter >= _skipQuantum)
{
// grow the skip pointer array if required
if (_skipPointerCount == _skipPointers.Length)
{
Array.Resize(ref _skipPointers, (int)Math.Ceiling(_skipPointers.Length * 1.5));
}
_skipPointers[_skipPointerCount++] = _upperBitCount + i + 1;
_negatedUnaryReadCounter = 0;
}
}
// store the delta of the upper bits in unary encoding
_upperBitCount += delta;
if (_upperBitCount / 32 >= _upperBits.Length)
{
Array.Resize(ref _upperBits, (int)Math.Ceiling(_upperBitCount * 1.5));
}
_upperBits[_upperBitCount / 32] |= (uint)(1 << (int)(_upperBitCount % 32));
_upperBitCount += 1;
// build the skip list for direct indexing
_unaryReadCounter += 1;
if (_unaryReadCounter > _indexQuantum)
{
if (_indexPointerCount == _indexPointers.Length)
{
Array.Resize(ref _indexPointers, (int)Math.Ceiling(_indexPointers.Length * 1.5));
}
_indexPointers[_indexPointerCount++] = _upperBitCount;
_unaryReadCounter = 1;
}
_count += 1;
_lastAddedItem = item;
}
}
