public void testsetSizeInBits() { Console.WriteLine("testing setSizeInBits"); for (int k = 0; k < 4096; ++k) { EwahCompressedBitArray ewah = new EwahCompressedBitArray(); ewah.SizeInBits = k; Assert.AreEqual(ewah.SizeInBits, k); Assert.AreEqual(ewah.GetCardinality(), 0); EwahCompressedBitArray ewah2 = new EwahCompressedBitArray(); ewah2.SetSizeInBits(k, false); Assert.AreEqual(ewah2.SizeInBits, k); Assert.AreEqual(ewah2.GetCardinality(), 0); EwahCompressedBitArray ewah3 = new EwahCompressedBitArray(); for (int i = 0; i < k; ++i) { ewah3.Set(i); } Assert.AreEqual(ewah3.SizeInBits, k); Assert.AreEqual(ewah3.GetCardinality(), k); EwahCompressedBitArray ewah4 = new EwahCompressedBitArray(); ewah4.SetSizeInBits(k, true); Assert.AreEqual(ewah4.SizeInBits, k); Assert.AreEqual(ewah4.GetCardinality(), k); } }
private EwahCompressedBitArray getFilledBitmap(bool fill) { EwahCompressedBitArray bitmap = new EwahCompressedBitArray(); bitmap.SetSizeInBits(_maxBitSize, fill); return(bitmap); }
public void TestSizeInBits1() { Console.WriteLine("testing TestSizeInBits1"); EwahCompressedBitArray bitmap = new EwahCompressedBitArray(); bitmap.SetSizeInBits(1, false); Assert.AreEqual(1, bitmap.SizeInBits); bitmap.Not(); Assert.AreEqual(1, bitmap.GetCardinality()); }
public void TestCloneEwahCompressedBitArray() { Console.WriteLine("testing EWAH clone"); EwahCompressedBitArray a = new EwahCompressedBitArray(); a.Set(410018); a.Set(410019); a.Set(410020); a.Set(410021); a.Set(410022); a.Set(410023); EwahCompressedBitArray b = (EwahCompressedBitArray)a.Clone(); a.SetSizeInBits(487123, false); b.SetSizeInBits(487123, false); Assert.AreEqual(a, b); }
public void testDebugSetSizeInBitsTest() { Console.WriteLine("testing DebugSetSizeInBits"); EwahCompressedBitArray b = new EwahCompressedBitArray(); b.Set(4); b.SetSizeInBits(6, true); List <int> positions = b.GetPositions(); Assert.AreEqual(2, positions.Count); Assert.AreEqual(4, positions[0]); Assert.AreEqual(5, positions[1]); IEnumerator <int> iterator = ((IEnumerable <int>)b).GetEnumerator(); Assert.AreEqual(true, iterator.MoveNext()); Assert.AreEqual(4, iterator.Current); Assert.AreEqual(true, iterator.MoveNext()); Assert.AreEqual(5, iterator.Current); Assert.AreEqual(false, iterator.MoveNext()); }
public EwahCompressedBitArray query(BICriteria criteria) { // This method could use recursion which would make it more readable // For performance reasons, and to avoid StackOverflowException, it // uses the Snapshot class to avoid recursion if (criteria == null) { throw new ArgumentNullException("criteria"); } EwahCompressedBitArray temp; Snapshot previous; Snapshot next; Snapshot current = new Snapshot(); current.criteria = criteria; current.state = 0; Stack <Snapshot> stack = new Stack <Snapshot>(); stack.Push(current); while (stack.Count > 0) { current = stack.Pop(); if (stack.Count > 0) { previous = stack.Peek(); } else { previous = null; } if (current.criteria.CriteriaOperator == BICriteria.Operator.OR || current.criteria.CriteriaOperator == BICriteria.Operator.AND) { if (current.state == 0) { current.state = 1; stack.Push(current); next = new Snapshot(); next.criteria = current.criteria.LeftCriteria; next.state = 0; stack.Push(next); continue; } else if (current.state == 1) { current.state = 2; stack.Push(current); next = new Snapshot(); next.criteria = current.criteria.RightCriteria; next.state = 0; stack.Push(next); continue; } else { if (current.criteria.CriteriaOperator == BICriteria.Operator.AND) { temp = (current.left.And(current.right)); } else { temp = (current.left.Or(current.right)); } if (previous == null) { return(temp); } else if (previous.state == 1) { previous.left = temp; } else if (previous.state == 2) { previous.right = temp; } continue; } } else { if (current.criteria.CriteriaOperator == BICriteria.Operator.EMPTY_ONLY) { if (previous == null) { return(getEmptyBitmap(current.criteria.Key)); } else if (previous.state == 1) { previous.left = getEmptyBitmap(current.criteria.Key); } else if (previous.state == 2) { previous.right = getEmptyBitmap(current.criteria.Key); } continue; } else { EwahCompressedBitArray bitmap; if (!_bitmaps.ContainsKey(current.criteria.Key)) { bitmap = getFilledBitmap(false); } else { bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]); } if (current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS || current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS_OR_EMPTY) { bitmap.Not(); } if (current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS_OR_EMPTY || current.criteria.CriteriaOperator == BICriteria.Operator.EQUALS_OR_EMPTY) { bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key)); } if (previous == null) { return(bitmap); } else if (previous.state == 1) { previous.left = bitmap; } else if (previous.state == 2) { previous.right = bitmap; } continue; } } } temp = new EwahCompressedBitArray(); temp.SetSizeInBits(_maxBitSize, false); return(temp); }
internal EwahCompressedBitArray CreateBitMapIndexForExcludedTags(CLR.HashSet <string> tagsToExclude, QueryType queryType, bool printLoggingMessages = false) { var bitMapTimer = Stopwatch.StartNew(); var tagLookupForQueryType = GetTagByQueryLookup(queryType); var collectIdsTimer = Stopwatch.StartNew(); var excludedQuestionIds = cache.Value.GetCachedHashSet(); foreach (var tag in tagsToExclude) { foreach (var id in tagLookupForQueryType[tag]) { excludedQuestionIds.Add(id); } } collectIdsTimer.Stop(); // At the end we need to have the BitMap Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded // That way we can efficiently apply the exclusions by ANDing this BitMap to the previous results var allQuestions = tagLookupForQueryType[TagServer.ALL_TAGS_KEY]; var setBitsTimer = Stopwatch.StartNew(); var bitMap = new EwahCompressedBitArray(); for (int index = 0; index < allQuestions.Length; index++) { if (excludedQuestionIds.Contains(allQuestions[index])) { var wasSet = bitMap.SetOptimised(index); // Set a bit where you CAN'T use a question if (wasSet == false) { Logger.LogStartupMessage("Error, unable to set bit {0:N0} (SizeInBits = {1:N0})", index, bitMap.SizeInBits); } } } setBitsTimer.Stop(); var tidyUpTimer = Stopwatch.StartNew(); bitMap.SetSizeInBits(questions.Count, defaultvalue: false); bitMap.Shrink(); tidyUpTimer.Stop(); bitMapTimer.Stop(); if (printLoggingMessages) { Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to collect {2:N0} Question Ids from {3:N0} Tags", collectIdsTimer.Elapsed, collectIdsTimer.ElapsedMilliseconds, excludedQuestionIds.Count, tagsToExclude.Count); Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to set {2:N0} bits", setBitsTimer.Elapsed, setBitsTimer.ElapsedMilliseconds, bitMap.GetCardinality()); Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to tidy-up the Bit Map (SetSizeInBits(..) and Shrink()), Size={2:N0} bytes ({3:N2} MB)", tidyUpTimer.Elapsed, tidyUpTimer.ElapsedMilliseconds, bitMap.SizeInBytes, bitMap.SizeInBytes / 1024.0 / 1024.0); using (Utils.SetConsoleColour(ConsoleColor.DarkYellow)) { Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) in TOTAL, made BitMap from {2:N0} Tags ({3:N0} Qu Ids), Cardinality={4:N0} ({5:N0})\n", bitMapTimer.Elapsed, bitMapTimer.ElapsedMilliseconds, tagsToExclude.Count, excludedQuestionIds.Count, bitMap.GetCardinality(), (ulong)questions.Count - bitMap.GetCardinality()); } } return(bitMap); }
public EwahCompressedBitArray query(BICriteria criteria) { // This method could use recursion which would make it more readable // For performance reasons, and to avoid StackOverflowException, it // uses the Snapshot class to avoid recursion if (criteria == null) throw new ArgumentNullException("criteria"); EwahCompressedBitArray temp; Snapshot previous; Snapshot next; Snapshot current = new Snapshot(); current.criteria = criteria; current.state = 0; Stack<Snapshot> stack = new Stack<Snapshot>(); stack.Push(current); while (stack.Count > 0) { current = stack.Pop(); if (stack.Count > 0) previous = stack.Peek(); else previous = null; if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.OR || current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND) { if (current.state == 0) { current.state = 1; stack.Push(current); next = new Snapshot(); next.criteria = current.criteria.LeftCriteria; next.state = 0; stack.Push(next); continue; } else if (current.state == 1) { current.state = 2; stack.Push(current); next = new Snapshot(); next.criteria = current.criteria.RightCriteria; next.state = 0; stack.Push(next); continue; } else { if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND) temp = (current.left.And(current.right)); else temp = (current.left.Or(current.right)); if (previous == null) { return temp; } else if (previous.state == 1) { previous.left = temp; } else if (previous.state == 2) { previous.right = temp; } continue; } } else { if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EMPTY_ONLY) { if (previous == null) return getEmptyBitmap(current.criteria.Key); else if (previous.state == 1) previous.left = getEmptyBitmap(current.criteria.Key); else if (previous.state == 2) previous.right = getEmptyBitmap(current.criteria.Key); continue; } else { EwahCompressedBitArray bitmap; if (!_bitmaps.ContainsKey(current.criteria.Key)) bitmap = getFilledBitmap(false); else bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]); if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS || current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY) bitmap.Not(); if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY || current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EQUALS_OR_EMPTY) bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key)); if (previous == null) return bitmap; else if (previous.state == 1) previous.left = bitmap; else if (previous.state == 2) previous.right = bitmap; continue; } } } temp = new EwahCompressedBitArray(); temp.SetSizeInBits(_maxBitSize, false); return temp; }
private EwahCompressedBitArray getFilledBitmap(bool fill) { EwahCompressedBitArray bitmap = new EwahCompressedBitArray(); bitmap.SetSizeInBits(_maxBitSize, fill); return bitmap; }