public void TestMassiveAndNot()
        {
            Console.WriteLine("testing massive and not");
            int N    = 1024;
            var ewah = new EwahCompressedBitArray[N];

            for (int k = 0; k < ewah.Length; ++k)
            {
                ewah[k] = new EwahCompressedBitArray();
            }
            for (int k = 0; k < 30000; ++k)
            {
                ewah[(k + 2 * k * k) % ewah.Length].Set(k);
            }
            EwahCompressedBitArray answer  = ewah[0];
            EwahCompressedBitArray answer2 = ewah[0];

            ;
            for (int k = 1; k < ewah.Length; ++k)
            {
                answer = answer.AndNot(ewah[k]);
                EwahCompressedBitArray copy = null;
                try
                {
                    copy = (EwahCompressedBitArray)ewah[k].Clone();
                    copy.Not();
                    answer2.And(copy);
                    assertEqualsPositions(answer.GetPositions(), answer2.GetPositions());
                }
                catch (InvalidOperationException e)
                {
                    Console.Error.WriteLine(e.StackTrace);
                }
            }
            Console.WriteLine("testing massive and not:ok");
        }
Example #2
0
        internal QueryResult Query(QueryInfo info, EwahCompressedBitArray exclusionBitMap = null, bool printLoggingMessages = false)
        {
            var bitMap         = GetTagByQueryBitMapLookup(info.Type);
            var questionLookup = GetTagByQueryLookup(info.Type)[TagServer.ALL_TAGS_KEY];

            // Calculating the Cardinality can be (is?) expensive, we don't want to do it in Queries unless we really need to!?
            bool calculateCardinality = true; // false

            if (printLoggingMessages)
            {
                Logger.Log("Tag \"{0}\" is in {1:N0} Questions, Tag \"{2}\" is in {3:N0} Questions",
                           info.Tag, allTags[info.Tag], info.OtherTag, allTags[info.OtherTag]);
            }

            //PrintResults(Enumerable.Range(0, questionLookup.Length), qu => questionLookup[qu], TagServer.ALL_TAGS_KEY, info.Type);
            //PrintResults(bitMap[info.Tag], qu => questionLookup[qu], info.Tag, info.Type);
            //PrintResults(bitMap[info.OtherTag], qu => questionLookup[qu], info.OtherTag, info.Type);

            var timer      = Stopwatch.StartNew();
            var tag1BitMap = bitMap[info.Tag];
            var tag2BitMap = bitMap[info.OtherTag];
            EwahCompressedBitArray bitMapResult = new EwahCompressedBitArray();

            switch (info.Operator)
            {
            case "AND":
                bitMapResult = tag1BitMap.And(tag2BitMap);
                break;

            case "AND-NOT":
                bitMapResult = tag1BitMap.AndNot(tag2BitMap);
                break;

            case "OR":
                bitMapResult = tag1BitMap.Or(tag2BitMap);
                break;

            case "OR-NOT":     //"i.e. .net+or+jquery-"
                bitMapResult = tag1BitMap.OrNot(tag2BitMap);
                break;

            // TODO Work out what a "NOT" query really means, the LINQ version was "result = tag1Query.Except(tag2Query)" (which is the same as AND-NOT?!)
            //case "NOT":
            //    var bitMapResult = (EwahCompressedBitArray)tag2BitMap.Clone();
            //    bitMapResult.Not();
            //    break;

            default:
                throw new InvalidOperationException(string.Format("Invalid operator specified: {0}", info.Operator ?? "<NULL>"));
            }

            if (exclusionBitMap != null)
            {
                ulong cardinalityBeforeExclusions = 0;
                if (printLoggingMessages)
                {
                    cardinalityBeforeExclusions = bitMapResult.GetCardinality();
                }

                // The Exclusiong BitMap is Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
                // That way we can efficiently apply the exclusions by ANDing this BitMap to the previous results
                var exclusionTimer = Stopwatch.StartNew();
                bitMapResult = bitMapResult.AndNot(exclusionBitMap);
                exclusionTimer.Stop();

                if (printLoggingMessages)
                {
                    if (calculateCardinality)
                    {
                        Logger.Log("Took {0,5:N2} ms to apply exclusion BitMap (Cardinality={1:N0}), Results Cardinality: Before={2:N0}, After={3:N0}",
                                   exclusionTimer.Elapsed.TotalMilliseconds, exclusionBitMap.GetCardinality(), cardinalityBeforeExclusions, bitMapResult.GetCardinality());
                    }
                    else
                    {
                        Logger.Log("Took {0,5:N2} ms to apply exclusion BitMap", exclusionTimer.Elapsed.TotalMilliseconds);
                    }
                }
            }

            var resultCollectionTimer = Stopwatch.StartNew();
            var result = bitMapResult.Skip(info.Skip)
                         .Take(info.PageSize)
                         .Select(i => questions[questionLookup[i]])
                         .ToList();

            resultCollectionTimer.Stop();
            if (printLoggingMessages)
            {
                Logger.Log("Took {0,5:N2} ms to collect the results", resultCollectionTimer.Elapsed.TotalMilliseconds);
            }

            timer.Stop();

            Results.AddData(timer.Elapsed.TotalMilliseconds.ToString("#.##"));

            if (printLoggingMessages)
            {
                using (Utils.SetConsoleColour(ConsoleColor.DarkYellow))
                {
                    if (calculateCardinality)
                    {
                        Logger.Log("Took {0,5:N2} ms in TOTAL to calculate \"{1} {2} {3}\", Got {4} results, (Result Cardinality={5:N0})",
                                   timer.Elapsed.TotalMilliseconds, info.Tag, info.Operator, info.OtherTag, result.Count, bitMapResult.GetCardinality());
                    }
                    else
                    {
                        Logger.Log("Took {0,5:N2} ms in TOTAL to calculate \"{1} {2} {3}\", Got {4} results",
                                   timer.Elapsed.TotalMilliseconds, info.Tag, info.Operator, info.OtherTag, result.Count);
                    }
                }
                //PrintResults(bitMapResult, qu => questionLookup[qu], string.Format("{0} {1} {2}", info.Tag, info.Operator, info.OtherTag), info.Type);
                Logger.Log();
            }

            return(new QueryResult
            {
                Questions = result,
                // TODO see if we can get meaningful numbers here, WITHOUT calling GetCardinality() (because it's expensive)
                //Counters = new Dictionary<string, int>
                //{
                //    { "TagCounter", tagCounter },
                //    { "OtherTagCounter", otherTagCounter },
                //    { "ExclusionCounter", exclusionCounter.Counter }
                //}
            });
        }
        /**
         * a non-deterministic test proposed by Marc Polizzi.
         *
         * @param maxlength the maximum uncompressed size of the bitmap
         */

        public static void PolizziTest(int maxlength)
        {
            Console.WriteLine("Polizzi test with max length = " + maxlength);
            for (int k = 0; k < 10000; k += 77)
            {
                var rnd          = new Random();
                var ewahBitmap1  = new EwahCompressedBitArray();
                var clrBitArray1 = new BitArray(10000);
                var ewahBitmap2  = new EwahCompressedBitArray();
                var clrBitArray2 = new BitArray(10000);
                int len          = rnd.Next(maxlength);
                for (int pos = 0; pos < len; pos++)
                {
                    // random *** number of bits set ***
                    if (rnd.Next(7) == 0)
                    {
                        // random *** increasing *** values
                        ewahBitmap1.Set(pos);
                        clrBitArray1.Set(pos, true);
                    }
                    if (rnd.Next(11) == 0)
                    {
                        // random *** increasing *** values
                        ewahBitmap2.Set(pos);
                        clrBitArray2.Set(pos, true);
                    }
                }
                assertEquals(clrBitArray1, ewahBitmap1);
                assertEquals(clrBitArray2, ewahBitmap2);
                // XOR
                {
                    EwahCompressedBitArray xorEwahBitmap = ewahBitmap1.Xor(ewahBitmap2);
                    var xorclrBitArray = (BitArray)clrBitArray1.Clone();
                    xorclrBitArray.Xor(clrBitArray2);
                    assertEquals(xorclrBitArray, xorEwahBitmap);
                }
                // AND
                {
                    EwahCompressedBitArray andEwahBitmap = ewahBitmap1.And(ewahBitmap2);
                    var andclrBitArray = (BitArray)clrBitArray1.Clone();
                    andclrBitArray.And(clrBitArray2);
                    assertEquals(andclrBitArray, andEwahBitmap);
                }
                // AND
                {
                    EwahCompressedBitArray andEwahBitmap = ewahBitmap2.And(ewahBitmap1);
                    var andclrBitArray = (BitArray)clrBitArray1.Clone();
                    andclrBitArray.And(clrBitArray2);
                    assertEquals(andclrBitArray, andEwahBitmap);
                }
                // AND NOT
                {
                    EwahCompressedBitArray andNotEwahBitmap = ewahBitmap1
                                                              .AndNot(ewahBitmap2);
                    var andNotclrBitArray = (BitArray)clrBitArray1.Clone();
                    andNotclrBitArray.AndNot(clrBitArray2);
                    assertEquals(andNotclrBitArray, andNotEwahBitmap);
                }
                // AND NOT
                {
                    EwahCompressedBitArray andNotEwahBitmap = ewahBitmap2
                                                              .AndNot(ewahBitmap1);
                    var andNotclrBitArray = (BitArray)clrBitArray2.Clone();
                    andNotclrBitArray.AndNot(clrBitArray1);
                    assertEquals(andNotclrBitArray, andNotEwahBitmap);
                }
                // OR
                {
                    EwahCompressedBitArray orEwahBitmap = ewahBitmap1.Or(ewahBitmap2);
                    var orclrBitArray = (BitArray)clrBitArray1.Clone();
                    orclrBitArray.Or(clrBitArray2);
                    assertEquals(orclrBitArray, orEwahBitmap);
                }
                // OR
                {
                    EwahCompressedBitArray orEwahBitmap = ewahBitmap2.Or(ewahBitmap1);
                    var orclrBitArray = (BitArray)clrBitArray1.Clone();
                    orclrBitArray.Or(clrBitArray2);
                    assertEquals(orclrBitArray, orEwahBitmap);
                }
            }
        }
Example #4
0
 private static EwahCompressedBitArray Minus(this EwahCompressedBitArray FromThis, EwahCompressedBitArray SubtractThis)
 {
     return(FromThis.AndNot(SubtractThis));
 }