public void testsetSizeInBits()
{
Console.WriteLine("testing setSizeInBits");
for (int k = 0; k < 4096; ++k)
{
EwahCompressedBitArray ewah = new EwahCompressedBitArray();
ewah.SizeInBits = k;
Assert.AreEqual(ewah.SizeInBits, k);
Assert.AreEqual(ewah.GetCardinality(), 0);
EwahCompressedBitArray ewah2 = new EwahCompressedBitArray();
ewah2.SetSizeInBits(k, false);
Assert.AreEqual(ewah2.SizeInBits, k);
Assert.AreEqual(ewah2.GetCardinality(), 0);
EwahCompressedBitArray ewah3 = new EwahCompressedBitArray();
for (int i = 0; i < k; ++i)
{
ewah3.Set(i);
}
Assert.AreEqual(ewah3.SizeInBits, k);
Assert.AreEqual(ewah3.GetCardinality(), k);
EwahCompressedBitArray ewah4 = new EwahCompressedBitArray();
ewah4.SetSizeInBits(k, true);
Assert.AreEqual(ewah4.SizeInBits, k);
Assert.AreEqual(ewah4.GetCardinality(), k);
}
}
private EwahCompressedBitArray getFilledBitmap(bool fill)
{
EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
bitmap.SetSizeInBits(_maxBitSize, fill);
return(bitmap);
}
public void TestSizeInBits1()
{
Console.WriteLine("testing TestSizeInBits1");
EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
bitmap.SetSizeInBits(1, false);
Assert.AreEqual(1, bitmap.SizeInBits);
bitmap.Not();
Assert.AreEqual(1, bitmap.GetCardinality());
}
public void TestCloneEwahCompressedBitArray()
{
Console.WriteLine("testing EWAH clone");
EwahCompressedBitArray a = new EwahCompressedBitArray();
a.Set(410018);
a.Set(410019);
a.Set(410020);
a.Set(410021);
a.Set(410022);
a.Set(410023);
EwahCompressedBitArray b = (EwahCompressedBitArray)a.Clone();
a.SetSizeInBits(487123, false);
b.SetSizeInBits(487123, false);
Assert.AreEqual(a, b);
}
public void testDebugSetSizeInBitsTest()
{
Console.WriteLine("testing DebugSetSizeInBits");
EwahCompressedBitArray b = new EwahCompressedBitArray();
b.Set(4);
b.SetSizeInBits(6, true);
List <int> positions = b.GetPositions();
Assert.AreEqual(2, positions.Count);
Assert.AreEqual(4, positions[0]);
Assert.AreEqual(5, positions[1]);
IEnumerator <int> iterator = ((IEnumerable <int>)b).GetEnumerator();
Assert.AreEqual(true, iterator.MoveNext());
Assert.AreEqual(4, iterator.Current);
Assert.AreEqual(true, iterator.MoveNext());
Assert.AreEqual(5, iterator.Current);
Assert.AreEqual(false, iterator.MoveNext());
}
public EwahCompressedBitArray query(BICriteria criteria)
{
// This method could use recursion which would make it more readable
// For performance reasons, and to avoid StackOverflowException, it
// uses the Snapshot class to avoid recursion
if (criteria == null)
{
throw new ArgumentNullException("criteria");
}
EwahCompressedBitArray temp;
Snapshot previous;
Snapshot next;
Snapshot current = new Snapshot();
current.criteria = criteria;
current.state = 0;
Stack <Snapshot> stack = new Stack <Snapshot>();
stack.Push(current);
while (stack.Count > 0)
{
current = stack.Pop();
if (stack.Count > 0)
{
previous = stack.Peek();
}
else
{
previous = null;
}
if (current.criteria.CriteriaOperator == BICriteria.Operator.OR ||
current.criteria.CriteriaOperator == BICriteria.Operator.AND)
{
if (current.state == 0)
{
current.state = 1;
stack.Push(current);
next = new Snapshot();
next.criteria = current.criteria.LeftCriteria;
next.state = 0;
stack.Push(next);
continue;
}
else if (current.state == 1)
{
current.state = 2;
stack.Push(current);
next = new Snapshot();
next.criteria = current.criteria.RightCriteria;
next.state = 0;
stack.Push(next);
continue;
}
else
{
if (current.criteria.CriteriaOperator == BICriteria.Operator.AND)
{
temp = (current.left.And(current.right));
}
else
{
temp = (current.left.Or(current.right));
}
if (previous == null)
{
return(temp);
}
else if (previous.state == 1)
{
previous.left = temp;
}
else if (previous.state == 2)
{
previous.right = temp;
}
continue;
}
}
else
{
if (current.criteria.CriteriaOperator == BICriteria.Operator.EMPTY_ONLY)
{
if (previous == null)
{
return(getEmptyBitmap(current.criteria.Key));
}
else if (previous.state == 1)
{
previous.left = getEmptyBitmap(current.criteria.Key);
}
else if (previous.state == 2)
{
previous.right = getEmptyBitmap(current.criteria.Key);
}
continue;
}
else
{
EwahCompressedBitArray bitmap;
if (!_bitmaps.ContainsKey(current.criteria.Key))
{
bitmap = getFilledBitmap(false);
}
else
{
bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]);
}
if (current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS ||
current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS_OR_EMPTY)
{
bitmap.Not();
}
if (current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS_OR_EMPTY ||
current.criteria.CriteriaOperator == BICriteria.Operator.EQUALS_OR_EMPTY)
{
bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key));
}
if (previous == null)
{
return(bitmap);
}
else if (previous.state == 1)
{
previous.left = bitmap;
}
else if (previous.state == 2)
{
previous.right = bitmap;
}
continue;
}
}
}
temp = new EwahCompressedBitArray();
temp.SetSizeInBits(_maxBitSize, false);
return(temp);
}
internal EwahCompressedBitArray CreateBitMapIndexForExcludedTags(CLR.HashSet <string> tagsToExclude, QueryType queryType, bool printLoggingMessages = false)
{
var bitMapTimer = Stopwatch.StartNew();
var tagLookupForQueryType = GetTagByQueryLookup(queryType);
var collectIdsTimer = Stopwatch.StartNew();
var excludedQuestionIds = cache.Value.GetCachedHashSet();
foreach (var tag in tagsToExclude)
{
foreach (var id in tagLookupForQueryType[tag])
{
excludedQuestionIds.Add(id);
}
}
collectIdsTimer.Stop();
// At the end we need to have the BitMap Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
// That way we can efficiently apply the exclusions by ANDing this BitMap to the previous results
var allQuestions = tagLookupForQueryType[TagServer.ALL_TAGS_KEY];
var setBitsTimer = Stopwatch.StartNew();
var bitMap = new EwahCompressedBitArray();
for (int index = 0; index < allQuestions.Length; index++)
{
if (excludedQuestionIds.Contains(allQuestions[index]))
{
var wasSet = bitMap.SetOptimised(index); // Set a bit where you CAN'T use a question
if (wasSet == false)
{
Logger.LogStartupMessage("Error, unable to set bit {0:N0} (SizeInBits = {1:N0})", index, bitMap.SizeInBits);
}
}
}
setBitsTimer.Stop();
var tidyUpTimer = Stopwatch.StartNew();
bitMap.SetSizeInBits(questions.Count, defaultvalue: false);
bitMap.Shrink();
tidyUpTimer.Stop();
bitMapTimer.Stop();
if (printLoggingMessages)
{
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to collect {2:N0} Question Ids from {3:N0} Tags",
collectIdsTimer.Elapsed, collectIdsTimer.ElapsedMilliseconds, excludedQuestionIds.Count, tagsToExclude.Count);
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to set {2:N0} bits",
setBitsTimer.Elapsed, setBitsTimer.ElapsedMilliseconds, bitMap.GetCardinality());
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to tidy-up the Bit Map (SetSizeInBits(..) and Shrink()), Size={2:N0} bytes ({3:N2} MB)",
tidyUpTimer.Elapsed, tidyUpTimer.ElapsedMilliseconds, bitMap.SizeInBytes, bitMap.SizeInBytes / 1024.0 / 1024.0);
using (Utils.SetConsoleColour(ConsoleColor.DarkYellow))
{
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) in TOTAL, made BitMap from {2:N0} Tags ({3:N0} Qu Ids), Cardinality={4:N0} ({5:N0})\n",
bitMapTimer.Elapsed, bitMapTimer.ElapsedMilliseconds,
tagsToExclude.Count,
excludedQuestionIds.Count,
bitMap.GetCardinality(),
(ulong)questions.Count - bitMap.GetCardinality());
}
}
return(bitMap);
}
public EwahCompressedBitArray query(BICriteria criteria)
{
// This method could use recursion which would make it more readable
// For performance reasons, and to avoid StackOverflowException, it
// uses the Snapshot class to avoid recursion
if (criteria == null)
throw new ArgumentNullException("criteria");
EwahCompressedBitArray temp;
Snapshot previous;
Snapshot next;
Snapshot current = new Snapshot();
current.criteria = criteria;
current.state = 0;
Stack<Snapshot> stack = new Stack<Snapshot>();
stack.Push(current);
while (stack.Count > 0)
{
current = stack.Pop();
if (stack.Count > 0)
previous = stack.Peek();
else
previous = null;
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.OR ||
current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
{
if (current.state == 0)
{
current.state = 1;
stack.Push(current);
next = new Snapshot();
next.criteria = current.criteria.LeftCriteria;
next.state = 0;
stack.Push(next);
continue;
}
else if (current.state == 1)
{
current.state = 2;
stack.Push(current);
next = new Snapshot();
next.criteria = current.criteria.RightCriteria;
next.state = 0;
stack.Push(next);
continue;
}
else
{
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
temp = (current.left.And(current.right));
else
temp = (current.left.Or(current.right));
if (previous == null)
{
return temp;
}
else if (previous.state == 1)
{
previous.left = temp;
}
else if (previous.state == 2)
{
previous.right = temp;
}
continue;
}
}
else
{
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EMPTY_ONLY)
{
if (previous == null)
return getEmptyBitmap(current.criteria.Key);
else if (previous.state == 1)
previous.left = getEmptyBitmap(current.criteria.Key);
else if (previous.state == 2)
previous.right = getEmptyBitmap(current.criteria.Key);
continue;
}
else
{
EwahCompressedBitArray bitmap;
if (!_bitmaps.ContainsKey(current.criteria.Key))
bitmap = getFilledBitmap(false);
else
bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]);
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS ||
current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY)
bitmap.Not();
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY ||
current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EQUALS_OR_EMPTY)
bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key));
if (previous == null)
return bitmap;
else if (previous.state == 1)
previous.left = bitmap;
else if (previous.state == 2)
previous.right = bitmap;
continue;
}
}
}
temp = new EwahCompressedBitArray();
temp.SetSizeInBits(_maxBitSize, false);
return temp;
}
private EwahCompressedBitArray getFilledBitmap(bool fill)
{
EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
bitmap.SetSizeInBits(_maxBitSize, fill);
return bitmap;
}
