public void TestMassiveAnd()
        {
            Console.WriteLine("testing massive logical and");
            var ewah = new EwahCompressedBitArray[1024];

            for (int k = 0; k < ewah.Length; ++k)
            {
                ewah[k] = new EwahCompressedBitArray();
            }
            for (int k = 0; k < 30000; ++k)
            {
                ewah[(k + 2 * k * k) % ewah.Length].Set(k);
            }
            EwahCompressedBitArray answer = ewah[0];

            for (int k = 1; k < ewah.Length; ++k)
            {
                answer = answer.And(ewah[k]);
            }
            // result should be empty
            if (answer.GetPositions().Count != 0)
            {
                Console.WriteLine(answer.ToDebugString());
            }
            Assert.IsTrue(answer.GetPositions().Count == 0);
            Console.WriteLine("testing massive logical and:ok");
        }
        public void EwahIteratorProblem()
        {
            Console.WriteLine("testing ArnonMoscona");
            var bitmap = new EwahCompressedBitArray();

            for (int i = 9434560; i <= 9435159; i++)
            {
                bitmap.Set(i);
            }

            List <int> v = bitmap.GetPositions();
            int        k = 0;

            foreach (int ival in bitmap)
            {
                Assert.AreEqual(ival, v[k++]);
            }
            Assert.AreEqual(k, v.Count);

            for (k = 2; k <= 1024; k *= 2)
            {
                int[] bitsToSet = CreateSortedIntArrayOfBitsToSet(k, 434455 + 5 * k);
                var   ewah      = new EwahCompressedBitArray();
                foreach (int i in bitsToSet)
                {
                    ewah.Set(i);
                }
                assertEqualsPositions(bitsToSet, ewah.GetPositions());
            }
        }
예제 #3
0
        internal void ValidateExclusionBitMap(EwahCompressedBitArray bitMapIndex, CLR.HashSet <string> expandedTagsNGrams, QueryType queryType)
        {
            // Exclusion BitMap is Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
            var questionLookup   = GetTagByQueryLookup(queryType)[TagServer.ALL_TAGS_KEY];
            var invalidQuestions = new List <Tuple <Question, string> >();
            var NOTbitMapIndex   = ((EwahCompressedBitArray)bitMapIndex.Clone());

            NOTbitMapIndex.Not();
            var positions = NOTbitMapIndex.GetPositions();

            foreach (var position in positions)
            {
                var question = questions[questionLookup[position]];
                foreach (var tag in question.Tags)
                {
                    if (expandedTagsNGrams.Contains(tag))
                    {
                        invalidQuestions.Add(Tuple.Create(question, tag));
                    }
                }
                // Sometimes the validitation locks up my laptop, this *seems* to make a difference?!
                Thread.Yield();
            }

            using (Utils.SetConsoleColour(ConsoleColor.Blue))
                Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for INVALID tags", positions.Count);

            if (invalidQuestions.Any())
            {
                using (Utils.SetConsoleColour(ConsoleColor.Red))
                    Logger.Log("ERROR Validating Exclusion Bit Map, {0:N0} questions should have been excluded",
                               invalidQuestions.Select(i => i.Item1.Id).Distinct().Count());

                foreach (var error in invalidQuestions)
                {
                    Logger.Log("  {0,8}: {1} -> {2}", error.Item1.Id, String.Join(", ", error.Item1.Tags), error.Item2);
                }
            }


            var expectedPositions = bitMapIndex.GetPositions();

            foreach (var position in expectedPositions)
            {
                var question = questions[questionLookup[position]];
                if (question.Tags.Any(t => expandedTagsNGrams.Contains(t)) == false)
                {
                    using (Utils.SetConsoleColour(ConsoleColor.Red))
                        Logger.Log("ERROR {0,8}: {1} -> didn't contain ANY excluded tags", question.Id, String.Join(", ", question.Tags));
                }
            }

            using (Utils.SetConsoleColour(ConsoleColor.Blue))
                Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for EXPECTED tags", expectedPositions.Count);

            Logger.Log();
        }
        /**
         * Convenience function to assess equality between a compressed BitArray
         * and an uncompressed BitArray
         *
         * @param x the compressed BitArray/bitmap
         * @param y the uncompressed BitArray/bitmap
         */

        private static void AreEqual(EwahCompressedBitArray x, BitArray y)
        {
            Assert.AreEqual(x.GetCardinality(), y.Cardinality());
            var positions = new List <int>();

            for (int ii = 0; ii < y.Count; ii++)
            {
                if (y[ii])
                {
                    positions.Add(ii);
                }
            }
            AreEqual(x.GetPositions(), positions);
        }
        public void TestSetGet()
        {
            Console.WriteLine("testing EWAH Set/get");
            var ewcb = new EwahCompressedBitArray();

            int[] val = { 5, 4400, 44600, 55400, 1000000 };
            for (int k = 0; k < val.Length; ++k)
            {
                ewcb.Set(val[k]);
            }
            List <int> result = ewcb.GetPositions();

            AreEqual(val, result);
            Console.WriteLine("testing EWAH Set/get:ok");
        }
        public void SsiYanKaiTest()
        {
            Console.WriteLine("testing SsiYanKaiTest");
            EwahCompressedBitArray a          = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008, 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017, 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026, 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035, 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044, 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053, 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062, 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, 40071, 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100);
            EwahCompressedBitArray b          = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 270000);
            HashSet <int>          aPositions = new HashSet <int>(a.GetPositions());
            int intersection                  = 0;
            EwahCompressedBitArray inter      = new EwahCompressedBitArray();
            HashSet <int>          bPositions = new HashSet <int>(b.GetPositions());

            foreach (int integer in bPositions)
            {
                if (aPositions.Contains(integer))
                {
                    inter.Set(integer);
                    ++intersection;
                }
            }
            EwahCompressedBitArray and2 = a.And(b);
            List <int>             l1   = inter.GetPositions();
            List <int>             l2   = and2.GetPositions();
            var ok = true;

            if (l1.Count != l2.Count)
            {
                Console.WriteLine("cardinality differs = " + l1.Count + " " + l2.Count);
                ok = false;
            }
            for (int k = 0; k < l1.Count; ++k)
            {
                if (l1[k] != l2[k])
                {
                    Console.WriteLine("differ at " + k + " = " + l1[k] + " " + l2[k]);
                    ok = false;
                }
            }
            Assert.IsTrue(ok);
            Assert.AreEqual(true, and2.Equals(inter));
            Assert.AreEqual(inter.GetHashCode(), and2.GetHashCode());
            Assert.AreEqual(intersection, and2.GetCardinality());
        }
        // part of a test contributed by Marc Polizzi

        /**
         * Assert equals positions.
         *
         * @param clrBitArray the jdk bitmap
         * @param ewahBitmap the ewah bitmap
         */

        private static void assertEqualsPositions(BitArray clrBitArray,
                                                  EwahCompressedBitArray ewahBitmap)
        {
            List <int> positions = ewahBitmap.GetPositions();

            foreach (int position in positions)
            {
                Assert.IsTrue(clrBitArray.Get(position),
                              "positions: BitArray got different bits");
            }
            var ps = new HashSet <int>(positions);

            for (int pos = clrBitArray.NextSetBit(0);
                 pos >= 0;
                 pos = clrBitArray
                       .NextSetBit(pos + 1))
            {
                Assert.IsTrue(ps.Contains(pos),
                              "positions: BitArray got different bits");
            }
        }
        public void testDebugSetSizeInBitsTest()
        {
            Console.WriteLine("testing DebugSetSizeInBits");
            EwahCompressedBitArray b = new EwahCompressedBitArray();

            b.Set(4);

            b.SetSizeInBits(6, true);

            List <int> positions = b.GetPositions();

            Assert.AreEqual(2, positions.Count);
            Assert.AreEqual(4, positions[0]);
            Assert.AreEqual(5, positions[1]);

            IEnumerator <int> iterator = ((IEnumerable <int>)b).GetEnumerator();

            Assert.AreEqual(true, iterator.MoveNext());
            Assert.AreEqual(4, iterator.Current);
            Assert.AreEqual(true, iterator.MoveNext());
            Assert.AreEqual(5, iterator.Current);
            Assert.AreEqual(false, iterator.MoveNext());
        }
        public void TestMassiveAndNot()
        {
            Console.WriteLine("testing massive and not");
            int N    = 1024;
            var ewah = new EwahCompressedBitArray[N];

            for (int k = 0; k < ewah.Length; ++k)
            {
                ewah[k] = new EwahCompressedBitArray();
            }
            for (int k = 0; k < 30000; ++k)
            {
                ewah[(k + 2 * k * k) % ewah.Length].Set(k);
            }
            EwahCompressedBitArray answer  = ewah[0];
            EwahCompressedBitArray answer2 = ewah[0];

            ;
            for (int k = 1; k < ewah.Length; ++k)
            {
                answer = answer.AndNot(ewah[k]);
                EwahCompressedBitArray copy = null;
                try
                {
                    copy = (EwahCompressedBitArray)ewah[k].Clone();
                    copy.Not();
                    answer2.And(copy);
                    assertEqualsPositions(answer.GetPositions(), answer2.GetPositions());
                }
                catch (InvalidOperationException e)
                {
                    Console.Error.WriteLine(e.StackTrace);
                }
            }
            Console.WriteLine("testing massive and not:ok");
        }
        public void TestExternalization()
        {
            Console.WriteLine("testing EWAH externalization");
            var ewcb = new EwahCompressedBitArray();

            int[] val = { 5, 4400, 44600, 55400, 1000000 };
            foreach (int t in val)
            {
                ewcb.Set(t);
            }

            var bos = new MemoryStream();
            var bf  = new BinaryFormatter();

            bf.Serialize(bos, ewcb);
            bos.Position = 0;

            ewcb = (EwahCompressedBitArray)bf.Deserialize(bos);

            List <int> result = ewcb.GetPositions();

            AreEqual(val, result);
            Console.WriteLine("testing EWAH externalization:ok");
        }
        public void TestEwahCompressedBitArray()
        {
            Console.WriteLine("testing EWAH (basic)");
            const long zero       = 0;
            const long specialval = 1L | (1L << 4) | (1L << 63);
            const long notzero    = ~zero;
            var        myarray1   = new EwahCompressedBitArray
            {
                zero, zero, zero, specialval, specialval, notzero, zero
            };

            Assert.AreEqual(myarray1.GetPositions().Count, 6 + 64);
            var myarray2 = new EwahCompressedBitArray();

            myarray2.Add(zero);
            myarray2.Add(specialval);
            myarray2.Add(specialval);
            myarray2.Add(notzero);
            myarray2.Add(zero);
            myarray2.Add(zero);
            myarray2.Add(zero);
            Assert.AreEqual(myarray2.GetPositions().Count, 6 + 64);
            List <int> data1     = myarray1.GetPositions();
            List <int> data2     = myarray2.GetPositions();
            var        logicalor = new List <int>();

            {
                var tmp = new HashSet <int>();
                tmp.AddRange(data1);
                tmp.AddRange(data2);
                logicalor.AddRange(tmp);
            }
            logicalor.Sort();
            var logicaland = new List <int>();

            logicaland.AddRange(data1);
            logicaland.Retain(data2);
            logicaland.Sort();
            EwahCompressedBitArray arrayand = myarray1.And(myarray2);

            AreEqual(arrayand.GetPositions(), logicaland);
            EwahCompressedBitArray arrayor = myarray1.Or(myarray2);

            AreEqual(arrayor.GetPositions(), logicalor);
            EwahCompressedBitArray arrayandbis = myarray2.And(myarray1);

            AreEqual(arrayandbis.GetPositions(), logicaland);
            EwahCompressedBitArray arrayorbis = myarray2.Or(myarray1);

            AreEqual(arrayorbis.GetPositions(), logicalor);
            var x = new EwahCompressedBitArray();

            foreach (int i in myarray1.GetPositions())
            {
                x.Set(i);
            }
            AreEqual(x.GetPositions(), myarray1.GetPositions());
            x = new EwahCompressedBitArray();
            foreach (int i in myarray2.GetPositions())
            {
                x.Set(i);
            }
            AreEqual(x.GetPositions(), myarray2.GetPositions());
            x = new EwahCompressedBitArray();
            foreach (int pos in myarray1)
            {
                x.Set(pos);
            }
            AreEqual(x.GetPositions(), myarray1.GetPositions());
            x = new EwahCompressedBitArray();
            foreach (int pos in myarray2)
            {
                x.Set(pos);
            }
            AreEqual(x.GetPositions(), myarray2.GetPositions());
            Console.WriteLine("testing EWAH (basic):ok");
        }