public void CompressionTest()
{
int indexedPages = 1000000000;
// may be increased to 1,000,000,000 (>2 GB) if: >=16 GB RAM, 64 bit Windows, .NET version >= 4.5, <gcAllowVeryLargeObjects> in config file, Project / Properties / Buld / Prefer 32-bit disabled!
// http://stackoverflow.com/questions/25225249/maxsize-of-array-in-net-framework-4-5
int maximumPostingListLength = 100000;
for (int postingListLength = 10; postingListLength <= maximumPostingListLength; postingListLength *= 10)
{
// posting list creation
Trace.WriteLine($"Create posting list with {postingListLength} items...");
List <uint> postingList1 = GeneratePostingList(postingListLength, indexedPages);
// compression
Trace.Write("Compress posting list");
var sw = Stopwatch.StartNew();
var compressedBuffer1 = new EliasFanoList((uint)indexedPages, maximumPostingListLength, postingList1);
Trace.WriteLine($" in {sw.ElapsedMilliseconds}ms...");
// decompression
Trace.Write("Decompress posting list");
sw.Restart();
var postingList10 = compressedBuffer1.ToList();
Trace.WriteLine($" in {sw.ElapsedMilliseconds}ms...");
// verification
Trace.WriteLine("Verify posting list...");
Assert.AreEqual(postingList1.Count, postingList10.Count);
for (int i = 0; i < postingList1.Count; i++)
{
Assert.AreEqual(postingList1[i], postingList10[i]);
}
}
}
public void EnumeratorSkipsBeyondLastBucket()
{
// compression
var compressedBuffer1 = new EliasFanoList(_postingList.Last(), _postingList.Count, _postingList);
// get enumerator and skip
var enumerator = compressedBuffer1.GetEnumerator() as ISkippingEnumerator <uint>;
Assert.AreEqual(false, enumerator.SkipToBoundary(50000000));
}
public void EnumeratorSkipsToLastItem()
{
// compression
var compressedBuffer1 = new EliasFanoList(_postingList.Last(), _postingList.Count, _postingList);
// get enumerator and skip
var enumerator = compressedBuffer1.GetEnumerator() as ISkippingEnumerator <uint>;
Assert.AreEqual(true, enumerator.SkipToBoundary(25070429));
Assert.AreEqual(25070429U, enumerator.Current);
}
public void RandomAccessReturnsCorrectResults()
{
// compression
var compressedBuffer1 = new EliasFanoList(_postingList.Last(), _postingList.Count, _postingList);
// decompression & verification
Assert.AreEqual(_postingList.Count, compressedBuffer1.Count);
for (int i = 0; i < _postingList.Count; i++)
{
Assert.AreEqual(_postingList[i], compressedBuffer1[i]);
}
}
public void EnumeratorSkipsToSecondPointerIndex()
{
// compression
var data = Enumerable.Range(0, 4096).Select(i => (uint)i).ToList();
var compressedBuffer1 = new EliasFanoList(data);
// get enumerator and skip
var enumerator = compressedBuffer1.GetEnumerator() as ISkippingEnumerator <uint>;
Assert.AreEqual(true, enumerator.SkipToIndex(2048));
Assert.AreEqual(2048U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(2049U, enumerator.Current);
}
public void EnumeratorReturnsCorrectResults()
{
// compression
var compressedBuffer1 = new EliasFanoList(_postingList.Last(), _postingList.Count, _postingList);
// decompression
var postingList10 = compressedBuffer1.ToList();
// verification
Assert.AreEqual(_postingList.Count, postingList10.Count);
for (int i = 0; i < _postingList.Count; i++)
{
Assert.AreEqual(_postingList[i], postingList10[i]);
}
}
public void EnumeratorSkipsByIndex()
{
// compression
var compressedBuffer1 = new EliasFanoList(_postingList.Last(), _postingList.Count, _postingList);
// get enumerator and skip
var enumerator = compressedBuffer1.GetEnumerator() as ISkippingEnumerator <uint>;
Assert.AreEqual(true, enumerator.SkipToIndex(10));
Assert.AreEqual(587702U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(587882U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(588062U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(588242U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(588422U, enumerator.Current);
}
public void EnumeratorSkipsBeyondBoundary()
{
// compression
var compressedBuffer1 = new EliasFanoList(_postingList.Last(), _postingList.Count, _postingList);
// get enumerator and skip
var enumerator = compressedBuffer1.GetEnumerator() as ISkippingEnumerator <uint>;
Assert.AreEqual(true, enumerator.SkipToBoundary(595621));
Assert.AreEqual(595622U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(595802U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(595982U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(596162U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(596342U, enumerator.Current);
Assert.AreEqual(true, enumerator.MoveNext());
Assert.AreEqual(596522U, enumerator.Current);
}
public void CanCompressLists()
{
var lists = new List <List <uint> >()
{
new List <uint>()
{
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20
},
new List <uint>()
{
87, 1009, 1117, 1221, 8065, 13647, 13954, 14057, 18688, 26675, 27183, 27966, 28069, 28426
},
new List <uint>()
{
12080479, 12082919, 12082936, 12093975, 49157819, 49164551
},
};
foreach (var list in lists)
{
var compressed = new EliasFanoList(list);
Assert.IsTrue(list.SequenceEqual(compressed));
}
}
