public void testsetSizeInBits()
 {
     Console.WriteLine("testing setSizeInBits");
     for (int k = 0; k < 4096; ++k)
     {
         EwahCompressedBitArray ewah = new EwahCompressedBitArray();
         ewah.SizeInBits = k;
         Assert.AreEqual(ewah.SizeInBits, k);
         Assert.AreEqual(ewah.GetCardinality(), 0);
         EwahCompressedBitArray ewah2 = new EwahCompressedBitArray();
         ewah2.SetSizeInBits(k, false);
         Assert.AreEqual(ewah2.SizeInBits, k);
         Assert.AreEqual(ewah2.GetCardinality(), 0);
         EwahCompressedBitArray ewah3 = new EwahCompressedBitArray();
         for (int i = 0; i < k; ++i)
         {
             ewah3.Set(i);
         }
         Assert.AreEqual(ewah3.SizeInBits, k);
         Assert.AreEqual(ewah3.GetCardinality(), k);
         EwahCompressedBitArray ewah4 = new EwahCompressedBitArray();
         ewah4.SetSizeInBits(k, true);
         Assert.AreEqual(ewah4.SizeInBits, k);
         Assert.AreEqual(ewah4.GetCardinality(), k);
     }
 }
Esempio n. 2
0
        private EwahCompressedBitArray getFilledBitmap(bool fill)
        {
            EwahCompressedBitArray bitmap = new EwahCompressedBitArray();

            bitmap.SetSizeInBits(_maxBitSize, fill);

            return(bitmap);
        }
        public void TestSizeInBits1()
        {
            Console.WriteLine("testing TestSizeInBits1");
            EwahCompressedBitArray bitmap = new EwahCompressedBitArray();

            bitmap.SetSizeInBits(1, false);
            Assert.AreEqual(1, bitmap.SizeInBits);
            bitmap.Not();
            Assert.AreEqual(1, bitmap.GetCardinality());
        }
        public void TestCloneEwahCompressedBitArray()
        {
            Console.WriteLine("testing EWAH clone");
            EwahCompressedBitArray a = new EwahCompressedBitArray();

            a.Set(410018);
            a.Set(410019);
            a.Set(410020);
            a.Set(410021);
            a.Set(410022);
            a.Set(410023);

            EwahCompressedBitArray b = (EwahCompressedBitArray)a.Clone();

            a.SetSizeInBits(487123, false);
            b.SetSizeInBits(487123, false);

            Assert.AreEqual(a, b);
        }
        public void testDebugSetSizeInBitsTest()
        {
            Console.WriteLine("testing DebugSetSizeInBits");
            EwahCompressedBitArray b = new EwahCompressedBitArray();

            b.Set(4);

            b.SetSizeInBits(6, true);

            List <int> positions = b.GetPositions();

            Assert.AreEqual(2, positions.Count);
            Assert.AreEqual(4, positions[0]);
            Assert.AreEqual(5, positions[1]);

            IEnumerator <int> iterator = ((IEnumerable <int>)b).GetEnumerator();

            Assert.AreEqual(true, iterator.MoveNext());
            Assert.AreEqual(4, iterator.Current);
            Assert.AreEqual(true, iterator.MoveNext());
            Assert.AreEqual(5, iterator.Current);
            Assert.AreEqual(false, iterator.MoveNext());
        }
Esempio n. 6
0
        public EwahCompressedBitArray query(BICriteria criteria)
        {
            // This method could use recursion which would make it more readable
            // For performance reasons, and to avoid StackOverflowException, it
            // uses the Snapshot class to avoid recursion

            if (criteria == null)
            {
                throw new ArgumentNullException("criteria");
            }

            EwahCompressedBitArray temp;
            Snapshot previous;
            Snapshot next;

            Snapshot current = new Snapshot();

            current.criteria = criteria;
            current.state    = 0;

            Stack <Snapshot> stack = new Stack <Snapshot>();

            stack.Push(current);

            while (stack.Count > 0)
            {
                current = stack.Pop();

                if (stack.Count > 0)
                {
                    previous = stack.Peek();
                }
                else
                {
                    previous = null;
                }

                if (current.criteria.CriteriaOperator == BICriteria.Operator.OR ||
                    current.criteria.CriteriaOperator == BICriteria.Operator.AND)
                {
                    if (current.state == 0)
                    {
                        current.state = 1;
                        stack.Push(current);

                        next          = new Snapshot();
                        next.criteria = current.criteria.LeftCriteria;
                        next.state    = 0;
                        stack.Push(next);

                        continue;
                    }
                    else if (current.state == 1)
                    {
                        current.state = 2;
                        stack.Push(current);

                        next          = new Snapshot();
                        next.criteria = current.criteria.RightCriteria;
                        next.state    = 0;
                        stack.Push(next);

                        continue;
                    }
                    else
                    {
                        if (current.criteria.CriteriaOperator == BICriteria.Operator.AND)
                        {
                            temp = (current.left.And(current.right));
                        }
                        else
                        {
                            temp = (current.left.Or(current.right));
                        }

                        if (previous == null)
                        {
                            return(temp);
                        }
                        else if (previous.state == 1)
                        {
                            previous.left = temp;
                        }
                        else if (previous.state == 2)
                        {
                            previous.right = temp;
                        }

                        continue;
                    }
                }
                else
                {
                    if (current.criteria.CriteriaOperator == BICriteria.Operator.EMPTY_ONLY)
                    {
                        if (previous == null)
                        {
                            return(getEmptyBitmap(current.criteria.Key));
                        }
                        else if (previous.state == 1)
                        {
                            previous.left = getEmptyBitmap(current.criteria.Key);
                        }
                        else if (previous.state == 2)
                        {
                            previous.right = getEmptyBitmap(current.criteria.Key);
                        }

                        continue;
                    }
                    else
                    {
                        EwahCompressedBitArray bitmap;

                        if (!_bitmaps.ContainsKey(current.criteria.Key))
                        {
                            bitmap = getFilledBitmap(false);
                        }
                        else
                        {
                            bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]);
                        }

                        if (current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS ||
                            current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS_OR_EMPTY)
                        {
                            bitmap.Not();
                        }

                        if (current.criteria.CriteriaOperator == BICriteria.Operator.NOT_EQUALS_OR_EMPTY ||
                            current.criteria.CriteriaOperator == BICriteria.Operator.EQUALS_OR_EMPTY)
                        {
                            bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key));
                        }

                        if (previous == null)
                        {
                            return(bitmap);
                        }
                        else if (previous.state == 1)
                        {
                            previous.left = bitmap;
                        }
                        else if (previous.state == 2)
                        {
                            previous.right = bitmap;
                        }

                        continue;
                    }
                }
            }

            temp = new EwahCompressedBitArray();
            temp.SetSizeInBits(_maxBitSize, false);
            return(temp);
        }
        internal EwahCompressedBitArray CreateBitMapIndexForExcludedTags(CLR.HashSet <string> tagsToExclude, QueryType queryType, bool printLoggingMessages = false)
        {
            var bitMapTimer = Stopwatch.StartNew();

            var tagLookupForQueryType = GetTagByQueryLookup(queryType);
            var collectIdsTimer       = Stopwatch.StartNew();
            var excludedQuestionIds   = cache.Value.GetCachedHashSet();

            foreach (var tag in tagsToExclude)
            {
                foreach (var id in tagLookupForQueryType[tag])
                {
                    excludedQuestionIds.Add(id);
                }
            }
            collectIdsTimer.Stop();

            // At the end we need to have the BitMap Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
            // That way we can efficiently apply the exclusions by ANDing this BitMap to the previous results

            var allQuestions = tagLookupForQueryType[TagServer.ALL_TAGS_KEY];
            var setBitsTimer = Stopwatch.StartNew();
            var bitMap       = new EwahCompressedBitArray();

            for (int index = 0; index < allQuestions.Length; index++)
            {
                if (excludedQuestionIds.Contains(allQuestions[index]))
                {
                    var wasSet = bitMap.SetOptimised(index); // Set a bit where you CAN'T use a question
                    if (wasSet == false)
                    {
                        Logger.LogStartupMessage("Error, unable to set bit {0:N0} (SizeInBits = {1:N0})", index, bitMap.SizeInBits);
                    }
                }
            }
            setBitsTimer.Stop();

            var tidyUpTimer = Stopwatch.StartNew();

            bitMap.SetSizeInBits(questions.Count, defaultvalue: false);
            bitMap.Shrink();
            tidyUpTimer.Stop();

            bitMapTimer.Stop();

            if (printLoggingMessages)
            {
                Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to collect {2:N0} Question Ids from {3:N0} Tags",
                                         collectIdsTimer.Elapsed, collectIdsTimer.ElapsedMilliseconds, excludedQuestionIds.Count, tagsToExclude.Count);
                Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to set {2:N0} bits",
                                         setBitsTimer.Elapsed, setBitsTimer.ElapsedMilliseconds, bitMap.GetCardinality());
                Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to tidy-up the Bit Map (SetSizeInBits(..) and Shrink()), Size={2:N0} bytes ({3:N2} MB)",
                                         tidyUpTimer.Elapsed, tidyUpTimer.ElapsedMilliseconds, bitMap.SizeInBytes, bitMap.SizeInBytes / 1024.0 / 1024.0);

                using (Utils.SetConsoleColour(ConsoleColor.DarkYellow))
                {
                    Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) in TOTAL, made BitMap from {2:N0} Tags ({3:N0} Qu Ids), Cardinality={4:N0} ({5:N0})\n",
                                             bitMapTimer.Elapsed, bitMapTimer.ElapsedMilliseconds,
                                             tagsToExclude.Count,
                                             excludedQuestionIds.Count,
                                             bitMap.GetCardinality(),
                                             (ulong)questions.Count - bitMap.GetCardinality());
                }
            }

            return(bitMap);
        }
        public EwahCompressedBitArray query(BICriteria criteria)
        {
            // This method could use recursion which would make it more readable
            // For performance reasons, and to avoid StackOverflowException, it
            // uses the Snapshot class to avoid recursion

            if (criteria == null)
                throw new ArgumentNullException("criteria");

            EwahCompressedBitArray temp;
            Snapshot previous;
            Snapshot next;

            Snapshot current = new Snapshot();
            current.criteria = criteria;
            current.state = 0;

            Stack<Snapshot> stack = new Stack<Snapshot>();
            stack.Push(current);

            while (stack.Count > 0)
            {
                current = stack.Pop();

                if (stack.Count > 0)
                    previous = stack.Peek();
                else
                    previous = null;

                if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.OR ||
                    current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
                {
                    if (current.state == 0)
                    {
                        current.state = 1;
                        stack.Push(current);

                        next = new Snapshot();
                        next.criteria = current.criteria.LeftCriteria;
                        next.state = 0;
                        stack.Push(next);

                        continue;
                    }
                    else if (current.state == 1)
                    {
                        current.state = 2;
                        stack.Push(current);

                        next = new Snapshot();
                        next.criteria = current.criteria.RightCriteria;
                        next.state = 0;
                        stack.Push(next);

                        continue;
                    }
                    else
                    {
                        if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
                            temp = (current.left.And(current.right));
                        else
                            temp = (current.left.Or(current.right));

                        if (previous == null)
                        {
                            return temp;
                        }
                        else if (previous.state == 1)
                        {
                            previous.left = temp;
                        }
                        else if (previous.state == 2)
                        {
                            previous.right = temp;
                        }

                        continue;
                    }
                }
                else
                {
                    if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EMPTY_ONLY)
                    {
                        if (previous == null)
                            return getEmptyBitmap(current.criteria.Key);
                        else if (previous.state == 1)
                            previous.left = getEmptyBitmap(current.criteria.Key);
                        else if (previous.state == 2)
                            previous.right = getEmptyBitmap(current.criteria.Key);

                        continue;
                    }
                    else
                    {
                        EwahCompressedBitArray bitmap;

                        if (!_bitmaps.ContainsKey(current.criteria.Key))
                            bitmap = getFilledBitmap(false);
                        else
                            bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]);

                        if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS ||
                            current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY)
                            bitmap.Not();

                        if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY ||
                            current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EQUALS_OR_EMPTY)
                            bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key));

                        if (previous == null)
                            return bitmap;
                        else if (previous.state == 1)
                            previous.left = bitmap;
                        else if (previous.state == 2)
                            previous.right = bitmap;

                        continue;
                    }
                }
            }

            temp = new EwahCompressedBitArray();
            temp.SetSizeInBits(_maxBitSize, false);
            return temp;
        }
        private EwahCompressedBitArray getFilledBitmap(bool fill)
        {
            EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
            bitmap.SetSizeInBits(_maxBitSize, fill);

            return bitmap;
        }