public void EwahIteratorProblem()
        {
            Console.WriteLine("testing ArnonMoscona");
            var bitmap = new EwahCompressedBitArray();

            for (int i = 9434560; i <= 9435159; i++)
            {
                bitmap.Set(i);
            }

            List <int> v = bitmap.GetPositions();
            int        k = 0;

            foreach (int ival in bitmap)
            {
                Assert.AreEqual(ival, v[k++]);
            }
            Assert.AreEqual(k, v.Count);

            for (k = 2; k <= 1024; k *= 2)
            {
                int[] bitsToSet = CreateSortedIntArrayOfBitsToSet(k, 434455 + 5 * k);
                var   ewah      = new EwahCompressedBitArray();
                foreach (int i in bitsToSet)
                {
                    ewah.Set(i);
                }
                assertEqualsPositions(bitsToSet, ewah.GetPositions());
            }
        }
예제 #2
0
    public static void Main(string[] args)
    {
        //RunAllTests();

        var ewahBitmap1                  = EwahCompressedBitArray.BitmapOf(345, 100987, 4309222);
        var ewahBitmap1_clone            = EwahCompressedBitArray.BitmapOf(345, 100987, 4309222);
        var ewahBitmap1_strictSubset     = EwahCompressedBitArray.BitmapOf(345, 100987);
        var ewahBitmap1_notstrictSubset2 = EwahCompressedBitArray.BitmapOf(100987, 7007);
        var ewahBitmap1_intersects_1     = EwahCompressedBitArray.BitmapOf(345, 67773, 100987);
        var ewahBitmap1_intersects_2     = EwahCompressedBitArray.BitmapOf(65, 345);
        var ewahBitmap1_3                = EwahCompressedBitArray.BitmapOf(55, 900000008);

        Console.WriteLine(ewahBitmap1_clone.IsSubsetOf(ewahBitmap1));
        Console.WriteLine(ewahBitmap1.IsSubsetOf(ewahBitmap1_clone));
        Console.WriteLine(ewahBitmap1.Equals(ewahBitmap1_clone));
        Console.WriteLine(ewahBitmap1_strictSubset.IsSubsetOf(ewahBitmap1));
        Console.WriteLine(ewahBitmap1_notstrictSubset2.IsSubsetOf(ewahBitmap1));
        Console.WriteLine(ewahBitmap1_intersects_1.IsSubsetOf(ewahBitmap1));
        Console.WriteLine(ewahBitmap1_intersects_2.IsSubsetOf(ewahBitmap1));

        Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_strictSubset));
        Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_intersects_1));
        Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_intersects_2));
        Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_3));
    }
예제 #3
0
        private static void TestBitMapIndexAndValidateResults(TagServer tagServer, QueryInfo queryInfo,
                                                              CLR.HashSet <string> tagsToExclude     = null,
                                                              EwahCompressedBitArray exclusionBitMap = null)
        {
            var result = tagServer.ComparisionQueryBitMapIndex(queryInfo, exclusionBitMap, printLoggingMessages: true);
            var errors = tagServer.GetInvalidResults(result.Questions, queryInfo);

            if (errors.Any())
            {
                using (Utils.SetConsoleColour(ConsoleColor.Red))
                    Logger.Log("ERROR Running \"{0}\" Query, {1} (out of {2}) results were invalid",
                               queryInfo.Operator, errors.Count, result.Questions.Count);
                foreach (var qu in errors)
                {
                    Logger.Log("  {0,8}: {1}", qu.Id, String.Join(", ", qu.Tags));
                }
                Logger.Log();
            }

            if (tagsToExclude != null && exclusionBitMap != null)
            {
                var shouldHaveBeenExcluded = tagServer.GetShouldHaveBeenExcludedResults(result.Questions, queryInfo, tagsToExclude);
                if (shouldHaveBeenExcluded.Any())
                {
                    using (Utils.SetConsoleColour(ConsoleColor.Red))
                        Logger.Log("ERROR Running \"{0}\" Query, {1} (out of {2}) questions should have been excluded",
                                   queryInfo.Operator, shouldHaveBeenExcluded.Select(s => s.Item1.Id).Distinct().Count(), result.Questions.Count);
                    foreach (var error in shouldHaveBeenExcluded)
                    {
                        Logger.Log("  {0,8}: {1} -> {2}", error.Item1.Id, String.Join(", ", error.Item1.Tags), string.Join(", ", error.Item2));
                    }
                    Logger.Log();
                }
            }
        }
 public void testsetSizeInBits()
 {
     Console.WriteLine("testing setSizeInBits");
     for (int k = 0; k < 4096; ++k)
     {
         EwahCompressedBitArray ewah = new EwahCompressedBitArray();
         ewah.SizeInBits = k;
         Assert.AreEqual(ewah.SizeInBits, k);
         Assert.AreEqual(ewah.GetCardinality(), 0);
         EwahCompressedBitArray ewah2 = new EwahCompressedBitArray();
         ewah2.SetSizeInBits(k, false);
         Assert.AreEqual(ewah2.SizeInBits, k);
         Assert.AreEqual(ewah2.GetCardinality(), 0);
         EwahCompressedBitArray ewah3 = new EwahCompressedBitArray();
         for (int i = 0; i < k; ++i)
         {
             ewah3.Set(i);
         }
         Assert.AreEqual(ewah3.SizeInBits, k);
         Assert.AreEqual(ewah3.GetCardinality(), k);
         EwahCompressedBitArray ewah4 = new EwahCompressedBitArray();
         ewah4.SetSizeInBits(k, true);
         Assert.AreEqual(ewah4.SizeInBits, k);
         Assert.AreEqual(ewah4.GetCardinality(), k);
     }
 }
        public void TestMassiveAnd()
        {
            Console.WriteLine("testing massive logical and");
            var ewah = new EwahCompressedBitArray[1024];

            for (int k = 0; k < ewah.Length; ++k)
            {
                ewah[k] = new EwahCompressedBitArray();
            }
            for (int k = 0; k < 30000; ++k)
            {
                ewah[(k + 2 * k * k) % ewah.Length].Set(k);
            }
            EwahCompressedBitArray answer = ewah[0];

            for (int k = 1; k < ewah.Length; ++k)
            {
                answer = answer.And(ewah[k]);
            }
            // result should be empty
            if (answer.GetPositions().Count != 0)
            {
                Console.WriteLine(answer.ToDebugString());
            }
            Assert.IsTrue(answer.GetPositions().Count == 0);
            Console.WriteLine("testing massive logical and:ok");
        }
예제 #6
0
파일: BitmapIndex.cs 프로젝트: Lanayx/hlc2
        public void Set(BIKey key, int bit)
        {
            EwahCompressedBitArray bitmap;
            EwahCompressedBitArray emptyBitmap;

            if (_bitmaps.ContainsKey(key))
            {
                bitmap = _bitmaps[key];
            }
            else
            {
                bitmap = new EwahCompressedBitArray();
                _bitmaps.Add(key, bitmap);
            }

            if (_emptyBitmaps.ContainsKey(key.Group))
            {
                emptyBitmap = _emptyBitmaps[key.Group];
            }
            else
            {
                emptyBitmap = new EwahCompressedBitArray();
                _emptyBitmaps.Add(key.Group, emptyBitmap);
            }

            bitmap.Set(bit);

            emptyBitmap.Not();
            emptyBitmap.Set(bit);
            emptyBitmap.Not();

            _maxBitSize = (_maxBitSize < (bit + 1) ? (bit + 1) : _maxBitSize);
        }
예제 #7
0
파일: BitmapIndex.cs 프로젝트: Lanayx/hlc2
        private EwahCompressedBitArray getFilledBitmap(bool fill)
        {
            EwahCompressedBitArray bitmap = new EwahCompressedBitArray();

            bitmap.SetSizeInBits(_maxBitSize, fill);

            return(bitmap);
        }
예제 #8
0
파일: BitmapIndex.cs 프로젝트: Lanayx/hlc2
        private EwahCompressedBitArray getCopyBitmap(EwahCompressedBitArray bitmap)
        {
            var bit = (EwahCompressedBitArray)bitmap.Clone();

            bit.SetSizeInBits(_maxBitSize, false);

            return(bit);
        }
예제 #9
0
        internal void ValidateExclusionBitMap(EwahCompressedBitArray bitMapIndex, CLR.HashSet <string> expandedTagsNGrams, QueryType queryType)
        {
            // Exclusion BitMap is Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
            var questionLookup   = GetTagByQueryLookup(queryType)[TagServer.ALL_TAGS_KEY];
            var invalidQuestions = new List <Tuple <Question, string> >();
            var NOTbitMapIndex   = ((EwahCompressedBitArray)bitMapIndex.Clone());

            NOTbitMapIndex.Not();
            var positions = NOTbitMapIndex.GetPositions();

            foreach (var position in positions)
            {
                var question = questions[questionLookup[position]];
                foreach (var tag in question.Tags)
                {
                    if (expandedTagsNGrams.Contains(tag))
                    {
                        invalidQuestions.Add(Tuple.Create(question, tag));
                    }
                }
                // Sometimes the validitation locks up my laptop, this *seems* to make a difference?!
                Thread.Yield();
            }

            using (Utils.SetConsoleColour(ConsoleColor.Blue))
                Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for INVALID tags", positions.Count);

            if (invalidQuestions.Any())
            {
                using (Utils.SetConsoleColour(ConsoleColor.Red))
                    Logger.Log("ERROR Validating Exclusion Bit Map, {0:N0} questions should have been excluded",
                               invalidQuestions.Select(i => i.Item1.Id).Distinct().Count());

                foreach (var error in invalidQuestions)
                {
                    Logger.Log("  {0,8}: {1} -> {2}", error.Item1.Id, String.Join(", ", error.Item1.Tags), error.Item2);
                }
            }


            var expectedPositions = bitMapIndex.GetPositions();

            foreach (var position in expectedPositions)
            {
                var question = questions[questionLookup[position]];
                if (question.Tags.Any(t => expandedTagsNGrams.Contains(t)) == false)
                {
                    using (Utils.SetConsoleColour(ConsoleColor.Red))
                        Logger.Log("ERROR {0,8}: {1} -> didn't contain ANY excluded tags", question.Id, String.Join(", ", question.Tags));
                }
            }

            using (Utils.SetConsoleColour(ConsoleColor.Blue))
                Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for EXPECTED tags", expectedPositions.Count);

            Logger.Log();
        }
        public void TestCardinality()
        {
            Console.WriteLine("testing EWAH GetCardinality");
            var bitmap = new EwahCompressedBitArray();

            bitmap.Set(int.MaxValue);
            // Console.format("Total Items %d\n", bitmap.GetCardinality());
            Assert.AreEqual(bitmap.GetCardinality(), 1);
            Console.WriteLine("testing EWAH GetCardinality:ok");
        }
예제 #11
0
파일: helpers.cs 프로젝트: i-e-b/KVK
 public static char[] BinaryStringLeftToRight(EwahCompressedBitArray subject)
 {
     char[] padding = "0000000000000000000000000000000000000000000000000000000000000000".ToArray();
     foreach (var position in subject)
     {
         if (position>63)break;
         padding[position] = '1';
     }
     return padding;
 }
        public void TestSizeInBits1()
        {
            Console.WriteLine("testing TestSizeInBits1");
            EwahCompressedBitArray bitmap = new EwahCompressedBitArray();

            bitmap.SetSizeInBits(1, false);
            Assert.AreEqual(1, bitmap.SizeInBits);
            bitmap.Not();
            Assert.AreEqual(1, bitmap.GetCardinality());
        }
        public void TestHasNextSafe()
        {
            Console.WriteLine("testing TestHasNextSafe");
            EwahCompressedBitArray bitmap = new EwahCompressedBitArray();

            bitmap.Set(0);
            IEnumerator <int> it = ((IEnumerable <int>)bitmap).GetEnumerator();

            Assert.AreEqual(it.MoveNext(), true);
            Assert.AreEqual(0, it.Current);
        }
예제 #14
0
    public static void Main(string[] args)
    {
        var ewahBitmap1 = EwahCompressedBitArray.BitmapOf(0, 2, 64, 1 << 30);
        var ewahBitmap2 = EwahCompressedBitArray.BitmapOf(1, 3, 64, 1 << 30);

        Console.WriteLine("Running demo program:");
        Console.WriteLine("bitmap 1: " + ewahBitmap1);
        Console.WriteLine("bitmap 2:" + ewahBitmap2);
        EwahCompressedBitArray orbitmap = ewahBitmap1.Or(ewahBitmap2);

        Console.WriteLine();
        Console.WriteLine("bitmap 1 OR bitmap 2:" + orbitmap);
        Console.WriteLine("memory usage: " + orbitmap.SizeInBytes + " bytes");
        Console.WriteLine();
        EwahCompressedBitArray andbitmap = ewahBitmap1.And(ewahBitmap2);

        Console.WriteLine("bitmap 1 AND bitmap 2:" + andbitmap);
        Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
        EwahCompressedBitArray xorbitmap = ewahBitmap1.Xor(ewahBitmap2);

        Console.WriteLine("bitmap 1 XOR bitmap 2:" + xorbitmap);
        Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
        Console.WriteLine("End of demo.");
        Console.WriteLine("");

        var tr = new EwahCompressedBitArrayTest();

        tr.TestYnosa();
        tr.TestIntersectOddNess();
        tr.testsetSizeInBits();
        tr.SsiYanKaiTest();
        tr.testDebugSetSizeInBitsTest();
        tr.EwahIteratorProblem();
        tr.TayaraTest();
        tr.TestNot();
        tr.TestCardinality();
        tr.TestEwahCompressedBitArray();
        tr.TestExternalization();
        tr.TestLargeEwahCompressedBitArray();
        tr.TestMassiveAnd();
        tr.TestMassiveAndNot();
        tr.TestMassiveOr();
        tr.TestMassiveXOR();
        tr.HabermaasTest();
        tr.VanSchaikTest();
        tr.TestRunningLengthWord();
        tr.TestSizeInBits1();
        tr.TestHasNextSafe();
        tr.TestCloneEwahCompressedBitArray();
        tr.TestSetGet();
        tr.TestWithParameters();

        new EWAHCompressedBitArraySerializerTest().TestCustomSerializationStrategy();
    }
        public void TestMassiveOr()
        {
            Console.WriteLine("testing massive logical or (can take a couple of minutes)");
            int N = 128;

            for (int howmany = 512; howmany <= 10000; howmany *= 2)
            {
                var ewah = new EwahCompressedBitArray[N];
                var bset = new BitArray[N];
                int k;
                for (k = 0; k < ewah.Length; ++k)
                {
                    ewah[k] = new EwahCompressedBitArray();
                }
                for (k = 0; k < bset.Length; ++k)
                {
                    bset[k] = new BitArray(10000);
                }
                for (k = 0; k < N; ++k)
                {
                    assertEqualsPositions(bset[k], ewah[k]);
                }
                for (k = 0; k < howmany; ++k)
                {
                    ewah[(k + 2 * k * k) % ewah.Length].Set(k);
                    bset[(k + 2 * k * k) % ewah.Length].Set(k, true);
                }
                for (k = 0; k < N; ++k)
                {
                    assertEqualsPositions(bset[k], ewah[k]);
                }
                EwahCompressedBitArray answer = ewah[0];
                BitArray BitArrayanswer       = bset[0];
                for (k = 1; k < ewah.Length; ++k)
                {
                    EwahCompressedBitArray tmp = answer.Or(ewah[k]);
                    BitArrayanswer.Or(bset[k]);
                    answer = tmp;
                    assertEqualsPositions(BitArrayanswer, answer);
                }
                assertEqualsPositions(BitArrayanswer, answer);
                k = 0;
                foreach (int j in answer)
                {
                    if (k != j)
                    {
                        Console.WriteLine(answer.ToDebugString());
                    }
                    Assert.AreEqual(k, j);
                    k += 1;
                }
            }
            Console.WriteLine("testing massive logical or:ok");
        }
        public void TestYnosa()
        {
            Console.WriteLine("testing Ynosa");
            var a1 = new EwahCompressedBitArray();
            var a2 = new EwahCompressedBitArray();

            a1.Set(5);
            a1.Set(15);
            a2.Set(5);
            Assert.IsTrue(a1.Intersects(a2));
            Console.WriteLine("testing Ynosa:ok");
        }
 /// <summary>
 /// Serializes an instance of <see cref="Ewah.EwahCompressedBitArray"/> into the given stream
 /// </summary>
 /// <param name="serializationStream">The serialization stream.</param>
 /// <param name="bitArray">The bit array.</param>
 public void Serialize(Stream serializationStream, EwahCompressedBitArray bitArray)
 {
     // No actual need to call Shrink with this serialisation strategy, so we can avoid
     // mutating the source type (side-effects are bad ;) )
     serializationStream.Write( BitConverter.GetBytes(bitArray.SizeInBits), 0, 4 );
     serializationStream.Write( BitConverter.GetBytes(bitArray._ActualSizeInWords),0, 4 );
     serializationStream.Write(BitConverter.GetBytes(bitArray._Rlw.Position), 0, 4);
     for(int i=0; i< bitArray._ActualSizeInWords;i++) {
         serializationStream.Write(BitConverter.GetBytes(bitArray._Buffer[i]), 0, 8);
     }
     return;
 }
예제 #18
0
        public void TestCustomSerializationStrategy()
        {
            Console.WriteLine("testing Custom serialization strategy");

            // Create a compressed bit array, and randomly assign up to 20,000 bits to it.
            var bmp = new EwahCompressedBitArray();
            var r   = new Random();

            for (int i = 0; i < 23000; i++)
            {
                if (r.NextDouble() < 0.5)
                {
                    bmp.Set(i);
                }
            }

            byte[] originalDeserialized = null;
            byte[] newFormDeserialized  = null;
            EwahCompressedBitArray newFormReserialized  = null;
            EwahCompressedBitArray originalReserialized = null;

            // First de-serialize+ re-serialize 'normally'
            using (var ms = new MemoryStream()) {
                BinaryFormatter bf = new BinaryFormatter();
                bf.Serialize(ms, bmp);
                originalDeserialized = ms.ToArray();
                ms.Seek(0, SeekOrigin.Begin);
                originalReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
            }

            // Now de-serialize + re-serialize with the new form.
            using (var ms = new MemoryStream()) {
                EwahCompressedBitArraySerializer bf = new EwahCompressedBitArraySerializer();
                bf.Serialize(ms, bmp);
                newFormDeserialized = ms.ToArray();
                ms.Seek(0, SeekOrigin.Begin);
                newFormReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
            }

            // Assert that the new form is more compact than the original form.
            Assert.Less(newFormDeserialized.Length, originalDeserialized.Length);

            // Compare the 'normal' de-serialized + re-serialized form, against the original.
            Assert.AreEqual(bmp, originalReserialized);

            // Compare the 'new form' de-serialized + re-serialized form, against the original.
            Assert.AreEqual(bmp, newFormReserialized);

            // Compare the 'normal' de-serialized + re-serialized form, against the newly de-serialized + re-serialized form.
            Assert.AreEqual(newFormReserialized, originalReserialized);

            Console.WriteLine("testing Custom serialization strategy:ok");
        }
예제 #19
0
 /// <summary>
 /// Serializes an instance of <see cref="Ewah.EwahCompressedBitArray"/> into the given stream
 /// </summary>
 /// <param name="serializationStream">The serialization stream.</param>
 /// <param name="bitArray">The bit array.</param>
 public void Serialize(Stream serializationStream, EwahCompressedBitArray bitArray)
 {
     // No actual need to call Shrink with this serialisation strategy, so we can avoid
     // mutating the source type (side-effects are bad ;-) )
     serializationStream.Write(BitConverter.GetBytes(bitArray.SizeInBits), 0, 4);
     serializationStream.Write(BitConverter.GetBytes(bitArray._ActualSizeInWords), 0, 4);
     serializationStream.Write(BitConverter.GetBytes(bitArray._Rlw.Position), 0, 4);
     for (int i = 0; i < bitArray._ActualSizeInWords; i++)
     {
         serializationStream.Write(BitConverter.GetBytes(bitArray._Buffer[i]), 0, 8);
     }
     return;
 }
        public void TestLargeEwahCompressedBitArray()
        {
            Console.WriteLine("testing EWAH over a large array");
            var       myarray1 = new EwahCompressedBitArray();
            const int n        = 11000000;

            for (int i = 0; i < n; ++i)
            {
                myarray1.Set(i);
            }
            Assert.AreEqual(myarray1.SizeInBits, n);
            Console.WriteLine("testing EWAH over a large array:ok");
        }
예제 #21
0
 public static void Main(string[] args)
 {
     var ewahBitmap1 = new EwahCompressedBitArray();
     var ewahBitmap2 = new EwahCompressedBitArray();
     ewahBitmap1.Set(0);
     ewahBitmap1.Set(2);
     ewahBitmap1.Set(64);
     ewahBitmap1.Set(1 << 30);
     Console.WriteLine("Running demo program:");
     Console.WriteLine("bitmap 1:");
     foreach (int k in ewahBitmap1)
         Console.WriteLine(k);
     ewahBitmap2.Set(1);
     ewahBitmap2.Set(3);
     ewahBitmap2.Set(64);
     ewahBitmap2.Set(1 << 30);
     Console.WriteLine("bitmap 2:");
     foreach (int k in ewahBitmap2)
         Console.WriteLine(k);
     Console.WriteLine();
     Console.WriteLine("bitmap 1 OR bitmap 2:");
     EwahCompressedBitArray orbitmap = ewahBitmap1.Or(ewahBitmap2);
     foreach (int k in orbitmap)
         Console.WriteLine(k);
     Console.WriteLine("memory usage: " + orbitmap.SizeInBytes + " bytes");
     Console.WriteLine();
     Console.WriteLine("bitmap 1 AND bitmap 2:");
     EwahCompressedBitArray andbitmap = ewahBitmap1.And(ewahBitmap2);
     foreach (int k in andbitmap)
         Console.WriteLine(k);
     Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
     Console.WriteLine("bitmap 1 XOR bitmap 2:");
     EwahCompressedBitArray xorbitmap = ewahBitmap1.Xor(ewahBitmap2);
     foreach (int k in xorbitmap)
         Console.WriteLine(k);
     Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
     Console.WriteLine("End of demo.");
     Console.WriteLine("");
     var tr = new EwahCompressedBitArrayTest();
     tr.TestNot();
     tr.TestCardinality();
     tr.TestEwahCompressedBitArray();
     tr.TestExternalization();
     tr.TestLargeEwahCompressedBitArray();
     tr.TestMassiveAnd();
     tr.TestMassiveAndNot();
     tr.TestMassiveOr();
     tr.TestMassiveXOR();
     tr.HabermaasTest();
     tr.VanSchaikTest();
 }
        public void TestCustomSerializationStrategy()
        {
            Console.WriteLine("testing Custom serialization strategy");

            // Create a compressed bit array, and randomly assign up to 20,000 bits to it.
            var bmp = new EwahCompressedBitArray();
            var r= new Random();
            for (int i = 0; i < 23000; i++) {
                if (r.NextDouble() < 0.5) {
                    bmp.Set(i);
                }
            }

            byte[] originalDeserialized= null;
            byte[] newFormDeserialized= null;
            EwahCompressedBitArray newFormReserialized= null;
            EwahCompressedBitArray originalReserialized= null;

            // First de-serialize+ re-serialize 'normally'
            using (var ms = new MemoryStream()) {
                BinaryFormatter bf = new BinaryFormatter();
                bf.Serialize(ms, bmp);
                originalDeserialized = ms.ToArray();
                ms.Seek(0, SeekOrigin.Begin);
                originalReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
            }

            // Now de-serialize + re-serialize with the new form.
            using (var ms = new MemoryStream()) {
                EwahCompressedBitArraySerializer bf = new EwahCompressedBitArraySerializer();
                bf.Serialize(ms, bmp);
                newFormDeserialized = ms.ToArray();
                ms.Seek(0, SeekOrigin.Begin);
                newFormReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
            }

            // Assert that the new form is more compact than the original form.
            Assert.Less(newFormDeserialized.Length, originalDeserialized.Length);

            // Compare the 'normal' de-serialized + re-serialized form, against the original.
            Assert.AreEqual(bmp, originalReserialized);

            // Compare the 'new form' de-serialized + re-serialized form, against the original.
            Assert.AreEqual(bmp, newFormReserialized);

            // Compare the 'normal' de-serialized + re-serialized form, against the newly de-serialized + re-serialized form.
            Assert.AreEqual(newFormReserialized, originalReserialized);

            Console.WriteLine("testing Custom serialization strategy:ok");
        }
        public void TestIntersectOddNess()
        {
            Console.WriteLine("testing IntersectOddNess");
            var a1 = new EwahCompressedBitArray();
            var a2 = new EwahCompressedBitArray();

            a1.Set(12);
            a2.Set(0);
            a2.Set(1);
            a2.Set(4);
            a2.Set(14);
            Assert.IsFalse(a1.Intersects(a2));
            Console.WriteLine("testing IntersectOddNess:ok");
        }
        public void TestNot()
        {
            Console.WriteLine("testing not");
            var bmp = new EwahCompressedBitArray();

            for (int i = 0; i <= 184; i++)
            {
                bmp.Set(i);
            }
            Assert.AreEqual(185, bmp.GetCardinality());
            bmp.Not();
            Assert.AreEqual(0, bmp.GetCardinality());
            Console.WriteLine("testing not:ok");
        }
        /**
         * Convenience function to assess equality between a compressed BitArray
         * and an uncompressed BitArray
         *
         * @param x the compressed BitArray/bitmap
         * @param y the uncompressed BitArray/bitmap
         */

        private static void AreEqual(EwahCompressedBitArray x, BitArray y)
        {
            Assert.AreEqual(x.GetCardinality(), y.Cardinality());
            var positions = new List <int>();

            for (int ii = 0; ii < y.Count; ii++)
            {
                if (y[ii])
                {
                    positions.Add(ii);
                }
            }
            AreEqual(x.GetPositions(), positions);
        }
예제 #26
0
        public UniversalSubjectPosition2(SortedSet <long> allRITEIds)
        {
            CausesOfLoss = new HashSet <SymbolicValue>()
            {
                "EQ", "WS", "CS", "FL", "WT", "FR", "TR", "SH", "FF", "SL", "WI", "SU", "HA", "TO", "SW", "FZ", "IC", "SN", "WF", "TS"
            };
            ResolvedExposureTypes = new HashSet <int>(ExposureType.GetIndividualIntExposureTypes(ExposureType.EExposureType.Loss));

            AllRITEIds          = new EwahCompressedBitArray();
            OffsetForAllRITEIds = allRITEIds.First();
            foreach (long RITEId in allRITEIds)
            {
                AllRITEIds.Set((int)(RITEId - OffsetForAllRITEIds));
            }
        }
        public void TestSetGet()
        {
            Console.WriteLine("testing EWAH Set/get");
            var ewcb = new EwahCompressedBitArray();

            int[] val = { 5, 4400, 44600, 55400, 1000000 };
            for (int k = 0; k < val.Length; ++k)
            {
                ewcb.Set(val[k]);
            }
            List <int> result = ewcb.GetPositions();

            AreEqual(val, result);
            Console.WriteLine("testing EWAH Set/get:ok");
        }
        //

        /**
         * Assess equality between an uncompressed bitmap and a compressed one,
         * part of a test contributed by Marc Polizzi
         *
         * @param clrBitArray the clr BitArray
         * @param ewahBitmap the ewah BitArray
         */

        private static void assertEqualsIterator(BitArray clrBitArray, EwahCompressedBitArray ewahBitmap)
        {
            var positions = new List <int>();

            foreach (int bit in ewahBitmap)
            {
                Assert.IsTrue(clrBitArray.Get(bit), "enumerator: BitArray got different bits");
                positions.Add(bit);
            }

            for (int pos = clrBitArray.NextSetBit(0); pos >= 0; pos = clrBitArray.NextSetBit(pos + 1))
            {
                Assert.IsTrue(positions.Contains(pos), "enumerator: BitArray got different bits");
            }
        }
        /**
         * Pseudo-non-deterministic test inspired by S.J.vanSchaik.
         * (Yes, non-deterministic tests are bad, but the test is actually deterministic.)
         */

        /**
         * Pseudo-non-deterministic test inspired by Federico Fissore.
         *
         * @param length the number of set bits in a bitmap
         */

        private static void ShouldSetBits(int length)
        {
            Console.WriteLine("testing shouldSetBits " + length);
            int[] bitsToSet = CreateSortedIntArrayOfBitsToSet(length, 434222);
            var   ewah      = new EwahCompressedBitArray();

            Console.WriteLine(" ... setting " + bitsToSet.Length + " values");
            foreach (int i in bitsToSet)
            {
                ewah.Set(i);
            }
            Console.WriteLine(" ... verifying " + bitsToSet.Length + " values");
            AreEqual(ewah, bitsToSet);
            Console.WriteLine(" ... checking GetCardinality");
            Assert.AreEqual(bitsToSet.Length, ewah.GetCardinality());
        }
 public void TayaraTest()
 {
     Console.WriteLine("Tayara test");
     for (int offset = 64; offset < (1 << 30); offset *= 2)
     {
         EwahCompressedBitArray a = new EwahCompressedBitArray();
         EwahCompressedBitArray b = new EwahCompressedBitArray();
         for (int k = 0; k < 64; ++k)
         {
             a.Set(offset + k);
             b.Set(offset + k);
         }
         Assert.AreEqual(a.And(b).Equals(a), true);
         Assert.AreEqual(a.Or(b).Equals(a), true);
     }
 }
        public void TestCloneEwahCompressedBitArray()
        {
            Console.WriteLine("testing EWAH clone");
            EwahCompressedBitArray a = new EwahCompressedBitArray();

            a.Set(410018);
            a.Set(410019);
            a.Set(410020);
            a.Set(410021);
            a.Set(410022);
            a.Set(410023);

            EwahCompressedBitArray b = (EwahCompressedBitArray)a.Clone();

            a.SetSizeInBits(487123, false);
            b.SetSizeInBits(487123, false);

            Assert.AreEqual(a, b);
        }
        public void TestMassiveXOR()
        {
            Console.WriteLine("testing massive xor (can take a couple of minutes)");
            int N    = 16;
            var ewah = new EwahCompressedBitArray[N];
            var bset = new BitArray[N];

            for (int k = 0; k < ewah.Length; ++k)
            {
                ewah[k] = new EwahCompressedBitArray();
            }
            for (int k = 0; k < bset.Length; ++k)
            {
                bset[k] = new BitArray(30000);
            }
            for (int k = 0; k < 30000; ++k)
            {
                ewah[(k + 2 * k * k) % ewah.Length].Set(k);
                bset[(k + 2 * k * k) % ewah.Length].Set(k, true);
            }
            EwahCompressedBitArray answer = ewah[0];
            BitArray BitArrayanswer       = bset[0];

            for (int k = 1; k < ewah.Length; ++k)
            {
                answer = answer.Xor(ewah[k]);
                BitArrayanswer.Xor(bset[k]);
                assertEqualsPositions(BitArrayanswer, answer);
            }
            int k2 = 0;

            foreach (int j in answer)
            {
                if (k2 != j)
                {
                    Console.WriteLine(answer.ToDebugString());
                }
                Assert.AreEqual(k2, j);
                k2 += 1;
            }
            Console.WriteLine("testing massive xor:ok");
        }
        public void SsiYanKaiTest()
        {
            Console.WriteLine("testing SsiYanKaiTest");
            EwahCompressedBitArray a          = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008, 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017, 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026, 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035, 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044, 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053, 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062, 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, 40071, 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100);
            EwahCompressedBitArray b          = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 270000);
            HashSet <int>          aPositions = new HashSet <int>(a.GetPositions());
            int intersection                  = 0;
            EwahCompressedBitArray inter      = new EwahCompressedBitArray();
            HashSet <int>          bPositions = new HashSet <int>(b.GetPositions());

            foreach (int integer in bPositions)
            {
                if (aPositions.Contains(integer))
                {
                    inter.Set(integer);
                    ++intersection;
                }
            }
            EwahCompressedBitArray and2 = a.And(b);
            List <int>             l1   = inter.GetPositions();
            List <int>             l2   = and2.GetPositions();
            var ok = true;

            if (l1.Count != l2.Count)
            {
                Console.WriteLine("cardinality differs = " + l1.Count + " " + l2.Count);
                ok = false;
            }
            for (int k = 0; k < l1.Count; ++k)
            {
                if (l1[k] != l2[k])
                {
                    Console.WriteLine("differ at " + k + " = " + l1[k] + " " + l2[k]);
                    ok = false;
                }
            }
            Assert.IsTrue(ok);
            Assert.AreEqual(true, and2.Equals(inter));
            Assert.AreEqual(inter.GetHashCode(), and2.GetHashCode());
            Assert.AreEqual(intersection, and2.GetCardinality());
        }
        // part of a test contributed by Marc Polizzi

        /**
         * Assert equals positions.
         *
         * @param clrBitArray the jdk bitmap
         * @param ewahBitmap the ewah bitmap
         */

        private static void assertEqualsPositions(BitArray clrBitArray,
                                                  EwahCompressedBitArray ewahBitmap)
        {
            List <int> positions = ewahBitmap.GetPositions();

            foreach (int position in positions)
            {
                Assert.IsTrue(clrBitArray.Get(position),
                              "positions: BitArray got different bits");
            }
            var ps = new HashSet <int>(positions);

            for (int pos = clrBitArray.NextSetBit(0);
                 pos >= 0;
                 pos = clrBitArray
                       .NextSetBit(pos + 1))
            {
                Assert.IsTrue(ps.Contains(pos),
                              "positions: BitArray got different bits");
            }
        }
        private EwahCompressedBitArray getCopyBitmap(EwahCompressedBitArray bitmap)
        {
            var bit = (EwahCompressedBitArray)bitmap.Clone();
            bit.SetSizeInBits(_maxBitSize, false);

            return bit;
        }
        public EwahCompressedBitArray query(BICriteria criteria)
        {
            // This method could use recursion which would make it more readable
            // For performance reasons, and to avoid StackOverflowException, it
            // uses the Snapshot class to avoid recursion

            if (criteria == null)
                throw new ArgumentNullException("criteria");

            EwahCompressedBitArray temp;
            Snapshot previous;
            Snapshot next;

            Snapshot current = new Snapshot();
            current.criteria = criteria;
            current.state = 0;

            Stack<Snapshot> stack = new Stack<Snapshot>();
            stack.Push(current);

            while (stack.Count > 0)
            {
                current = stack.Pop();

                if (stack.Count > 0)
                    previous = stack.Peek();
                else
                    previous = null;

                if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.OR ||
                    current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
                {
                    if (current.state == 0)
                    {
                        current.state = 1;
                        stack.Push(current);

                        next = new Snapshot();
                        next.criteria = current.criteria.LeftCriteria;
                        next.state = 0;
                        stack.Push(next);

                        continue;
                    }
                    else if (current.state == 1)
                    {
                        current.state = 2;
                        stack.Push(current);

                        next = new Snapshot();
                        next.criteria = current.criteria.RightCriteria;
                        next.state = 0;
                        stack.Push(next);

                        continue;
                    }
                    else
                    {
                        if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
                            temp = (current.left.And(current.right));
                        else
                            temp = (current.left.Or(current.right));

                        if (previous == null)
                        {
                            return temp;
                        }
                        else if (previous.state == 1)
                        {
                            previous.left = temp;
                        }
                        else if (previous.state == 2)
                        {
                            previous.right = temp;
                        }

                        continue;
                    }
                }
                else
                {
                    if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EMPTY_ONLY)
                    {
                        if (previous == null)
                            return getEmptyBitmap(current.criteria.Key);
                        else if (previous.state == 1)
                            previous.left = getEmptyBitmap(current.criteria.Key);
                        else if (previous.state == 2)
                            previous.right = getEmptyBitmap(current.criteria.Key);

                        continue;
                    }
                    else
                    {
                        EwahCompressedBitArray bitmap;

                        if (!_bitmaps.ContainsKey(current.criteria.Key))
                            bitmap = getFilledBitmap(false);
                        else
                            bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]);

                        if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS ||
                            current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY)
                            bitmap.Not();

                        if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY ||
                            current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EQUALS_OR_EMPTY)
                            bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key));

                        if (previous == null)
                            return bitmap;
                        else if (previous.state == 1)
                            previous.left = bitmap;
                        else if (previous.state == 2)
                            previous.right = bitmap;

                        continue;
                    }
                }
            }

            temp = new EwahCompressedBitArray();
            temp.SetSizeInBits(_maxBitSize, false);
            return temp;
        }
        public void Set(BIKey key, int bit)
        {
            EwahCompressedBitArray bitmap;
            EwahCompressedBitArray emptyBitmap;

            if (_bitmaps.ContainsKey(key))
                bitmap = _bitmaps[key];
            else
            {
                bitmap = new EwahCompressedBitArray();
                _bitmaps.Add(key, bitmap);
            }

            if (_emptyBitmaps.ContainsKey(key.Group))
                emptyBitmap = _emptyBitmaps[key.Group];
            else
            {
                emptyBitmap = new EwahCompressedBitArray();
                _emptyBitmaps.Add(key.Group, emptyBitmap);
            }

            bitmap.Set(bit);

            emptyBitmap.Not();
            emptyBitmap.Set(bit);
            emptyBitmap.Not();

            _maxBitSize = (_maxBitSize < (bit + 1) ? (bit + 1) : _maxBitSize);
        }
        private EwahCompressedBitArray getFilledBitmap(bool fill)
        {
            EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
            bitmap.SetSizeInBits(_maxBitSize, fill);

            return bitmap;
        }