public void EwahIteratorProblem()
{
Console.WriteLine("testing ArnonMoscona");
var bitmap = new EwahCompressedBitArray();
for (int i = 9434560; i <= 9435159; i++)
{
bitmap.Set(i);
}
List <int> v = bitmap.GetPositions();
int k = 0;
foreach (int ival in bitmap)
{
Assert.AreEqual(ival, v[k++]);
}
Assert.AreEqual(k, v.Count);
for (k = 2; k <= 1024; k *= 2)
{
int[] bitsToSet = CreateSortedIntArrayOfBitsToSet(k, 434455 + 5 * k);
var ewah = new EwahCompressedBitArray();
foreach (int i in bitsToSet)
{
ewah.Set(i);
}
assertEqualsPositions(bitsToSet, ewah.GetPositions());
}
}
public static void Main(string[] args)
{
//RunAllTests();
var ewahBitmap1 = EwahCompressedBitArray.BitmapOf(345, 100987, 4309222);
var ewahBitmap1_clone = EwahCompressedBitArray.BitmapOf(345, 100987, 4309222);
var ewahBitmap1_strictSubset = EwahCompressedBitArray.BitmapOf(345, 100987);
var ewahBitmap1_notstrictSubset2 = EwahCompressedBitArray.BitmapOf(100987, 7007);
var ewahBitmap1_intersects_1 = EwahCompressedBitArray.BitmapOf(345, 67773, 100987);
var ewahBitmap1_intersects_2 = EwahCompressedBitArray.BitmapOf(65, 345);
var ewahBitmap1_3 = EwahCompressedBitArray.BitmapOf(55, 900000008);
Console.WriteLine(ewahBitmap1_clone.IsSubsetOf(ewahBitmap1));
Console.WriteLine(ewahBitmap1.IsSubsetOf(ewahBitmap1_clone));
Console.WriteLine(ewahBitmap1.Equals(ewahBitmap1_clone));
Console.WriteLine(ewahBitmap1_strictSubset.IsSubsetOf(ewahBitmap1));
Console.WriteLine(ewahBitmap1_notstrictSubset2.IsSubsetOf(ewahBitmap1));
Console.WriteLine(ewahBitmap1_intersects_1.IsSubsetOf(ewahBitmap1));
Console.WriteLine(ewahBitmap1_intersects_2.IsSubsetOf(ewahBitmap1));
Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_strictSubset));
Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_intersects_1));
Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_intersects_2));
Console.WriteLine(ewahBitmap1.Minus(ewahBitmap1_3));
}
private static void TestBitMapIndexAndValidateResults(TagServer tagServer, QueryInfo queryInfo,
CLR.HashSet <string> tagsToExclude = null,
EwahCompressedBitArray exclusionBitMap = null)
{
var result = tagServer.ComparisionQueryBitMapIndex(queryInfo, exclusionBitMap, printLoggingMessages: true);
var errors = tagServer.GetInvalidResults(result.Questions, queryInfo);
if (errors.Any())
{
using (Utils.SetConsoleColour(ConsoleColor.Red))
Logger.Log("ERROR Running \"{0}\" Query, {1} (out of {2}) results were invalid",
queryInfo.Operator, errors.Count, result.Questions.Count);
foreach (var qu in errors)
{
Logger.Log(" {0,8}: {1}", qu.Id, String.Join(", ", qu.Tags));
}
Logger.Log();
}
if (tagsToExclude != null && exclusionBitMap != null)
{
var shouldHaveBeenExcluded = tagServer.GetShouldHaveBeenExcludedResults(result.Questions, queryInfo, tagsToExclude);
if (shouldHaveBeenExcluded.Any())
{
using (Utils.SetConsoleColour(ConsoleColor.Red))
Logger.Log("ERROR Running \"{0}\" Query, {1} (out of {2}) questions should have been excluded",
queryInfo.Operator, shouldHaveBeenExcluded.Select(s => s.Item1.Id).Distinct().Count(), result.Questions.Count);
foreach (var error in shouldHaveBeenExcluded)
{
Logger.Log(" {0,8}: {1} -> {2}", error.Item1.Id, String.Join(", ", error.Item1.Tags), string.Join(", ", error.Item2));
}
Logger.Log();
}
}
}
public void testsetSizeInBits()
{
Console.WriteLine("testing setSizeInBits");
for (int k = 0; k < 4096; ++k)
{
EwahCompressedBitArray ewah = new EwahCompressedBitArray();
ewah.SizeInBits = k;
Assert.AreEqual(ewah.SizeInBits, k);
Assert.AreEqual(ewah.GetCardinality(), 0);
EwahCompressedBitArray ewah2 = new EwahCompressedBitArray();
ewah2.SetSizeInBits(k, false);
Assert.AreEqual(ewah2.SizeInBits, k);
Assert.AreEqual(ewah2.GetCardinality(), 0);
EwahCompressedBitArray ewah3 = new EwahCompressedBitArray();
for (int i = 0; i < k; ++i)
{
ewah3.Set(i);
}
Assert.AreEqual(ewah3.SizeInBits, k);
Assert.AreEqual(ewah3.GetCardinality(), k);
EwahCompressedBitArray ewah4 = new EwahCompressedBitArray();
ewah4.SetSizeInBits(k, true);
Assert.AreEqual(ewah4.SizeInBits, k);
Assert.AreEqual(ewah4.GetCardinality(), k);
}
}
public void TestMassiveAnd()
{
Console.WriteLine("testing massive logical and");
var ewah = new EwahCompressedBitArray[1024];
for (int k = 0; k < ewah.Length; ++k)
{
ewah[k] = new EwahCompressedBitArray();
}
for (int k = 0; k < 30000; ++k)
{
ewah[(k + 2 * k * k) % ewah.Length].Set(k);
}
EwahCompressedBitArray answer = ewah[0];
for (int k = 1; k < ewah.Length; ++k)
{
answer = answer.And(ewah[k]);
}
// result should be empty
if (answer.GetPositions().Count != 0)
{
Console.WriteLine(answer.ToDebugString());
}
Assert.IsTrue(answer.GetPositions().Count == 0);
Console.WriteLine("testing massive logical and:ok");
}
public void Set(BIKey key, int bit)
{
EwahCompressedBitArray bitmap;
EwahCompressedBitArray emptyBitmap;
if (_bitmaps.ContainsKey(key))
{
bitmap = _bitmaps[key];
}
else
{
bitmap = new EwahCompressedBitArray();
_bitmaps.Add(key, bitmap);
}
if (_emptyBitmaps.ContainsKey(key.Group))
{
emptyBitmap = _emptyBitmaps[key.Group];
}
else
{
emptyBitmap = new EwahCompressedBitArray();
_emptyBitmaps.Add(key.Group, emptyBitmap);
}
bitmap.Set(bit);
emptyBitmap.Not();
emptyBitmap.Set(bit);
emptyBitmap.Not();
_maxBitSize = (_maxBitSize < (bit + 1) ? (bit + 1) : _maxBitSize);
}
private EwahCompressedBitArray getFilledBitmap(bool fill)
{
EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
bitmap.SetSizeInBits(_maxBitSize, fill);
return(bitmap);
}
private EwahCompressedBitArray getCopyBitmap(EwahCompressedBitArray bitmap)
{
var bit = (EwahCompressedBitArray)bitmap.Clone();
bit.SetSizeInBits(_maxBitSize, false);
return(bit);
}
internal void ValidateExclusionBitMap(EwahCompressedBitArray bitMapIndex, CLR.HashSet <string> expandedTagsNGrams, QueryType queryType)
{
// Exclusion BitMap is Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
var questionLookup = GetTagByQueryLookup(queryType)[TagServer.ALL_TAGS_KEY];
var invalidQuestions = new List <Tuple <Question, string> >();
var NOTbitMapIndex = ((EwahCompressedBitArray)bitMapIndex.Clone());
NOTbitMapIndex.Not();
var positions = NOTbitMapIndex.GetPositions();
foreach (var position in positions)
{
var question = questions[questionLookup[position]];
foreach (var tag in question.Tags)
{
if (expandedTagsNGrams.Contains(tag))
{
invalidQuestions.Add(Tuple.Create(question, tag));
}
}
// Sometimes the validitation locks up my laptop, this *seems* to make a difference?!
Thread.Yield();
}
using (Utils.SetConsoleColour(ConsoleColor.Blue))
Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for INVALID tags", positions.Count);
if (invalidQuestions.Any())
{
using (Utils.SetConsoleColour(ConsoleColor.Red))
Logger.Log("ERROR Validating Exclusion Bit Map, {0:N0} questions should have been excluded",
invalidQuestions.Select(i => i.Item1.Id).Distinct().Count());
foreach (var error in invalidQuestions)
{
Logger.Log(" {0,8}: {1} -> {2}", error.Item1.Id, String.Join(", ", error.Item1.Tags), error.Item2);
}
}
var expectedPositions = bitMapIndex.GetPositions();
foreach (var position in expectedPositions)
{
var question = questions[questionLookup[position]];
if (question.Tags.Any(t => expandedTagsNGrams.Contains(t)) == false)
{
using (Utils.SetConsoleColour(ConsoleColor.Red))
Logger.Log("ERROR {0,8}: {1} -> didn't contain ANY excluded tags", question.Id, String.Join(", ", question.Tags));
}
}
using (Utils.SetConsoleColour(ConsoleColor.Blue))
Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for EXPECTED tags", expectedPositions.Count);
Logger.Log();
}
public void TestCardinality()
{
Console.WriteLine("testing EWAH GetCardinality");
var bitmap = new EwahCompressedBitArray();
bitmap.Set(int.MaxValue);
// Console.format("Total Items %d\n", bitmap.GetCardinality());
Assert.AreEqual(bitmap.GetCardinality(), 1);
Console.WriteLine("testing EWAH GetCardinality:ok");
}
public static char[] BinaryStringLeftToRight(EwahCompressedBitArray subject)
{
char[] padding = "0000000000000000000000000000000000000000000000000000000000000000".ToArray();
foreach (var position in subject)
{
if (position>63)break;
padding[position] = '1';
}
return padding;
}
public void TestSizeInBits1()
{
Console.WriteLine("testing TestSizeInBits1");
EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
bitmap.SetSizeInBits(1, false);
Assert.AreEqual(1, bitmap.SizeInBits);
bitmap.Not();
Assert.AreEqual(1, bitmap.GetCardinality());
}
public void TestHasNextSafe()
{
Console.WriteLine("testing TestHasNextSafe");
EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
bitmap.Set(0);
IEnumerator <int> it = ((IEnumerable <int>)bitmap).GetEnumerator();
Assert.AreEqual(it.MoveNext(), true);
Assert.AreEqual(0, it.Current);
}
public static void Main(string[] args)
{
var ewahBitmap1 = EwahCompressedBitArray.BitmapOf(0, 2, 64, 1 << 30);
var ewahBitmap2 = EwahCompressedBitArray.BitmapOf(1, 3, 64, 1 << 30);
Console.WriteLine("Running demo program:");
Console.WriteLine("bitmap 1: " + ewahBitmap1);
Console.WriteLine("bitmap 2:" + ewahBitmap2);
EwahCompressedBitArray orbitmap = ewahBitmap1.Or(ewahBitmap2);
Console.WriteLine();
Console.WriteLine("bitmap 1 OR bitmap 2:" + orbitmap);
Console.WriteLine("memory usage: " + orbitmap.SizeInBytes + " bytes");
Console.WriteLine();
EwahCompressedBitArray andbitmap = ewahBitmap1.And(ewahBitmap2);
Console.WriteLine("bitmap 1 AND bitmap 2:" + andbitmap);
Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
EwahCompressedBitArray xorbitmap = ewahBitmap1.Xor(ewahBitmap2);
Console.WriteLine("bitmap 1 XOR bitmap 2:" + xorbitmap);
Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
Console.WriteLine("End of demo.");
Console.WriteLine("");
var tr = new EwahCompressedBitArrayTest();
tr.TestYnosa();
tr.TestIntersectOddNess();
tr.testsetSizeInBits();
tr.SsiYanKaiTest();
tr.testDebugSetSizeInBitsTest();
tr.EwahIteratorProblem();
tr.TayaraTest();
tr.TestNot();
tr.TestCardinality();
tr.TestEwahCompressedBitArray();
tr.TestExternalization();
tr.TestLargeEwahCompressedBitArray();
tr.TestMassiveAnd();
tr.TestMassiveAndNot();
tr.TestMassiveOr();
tr.TestMassiveXOR();
tr.HabermaasTest();
tr.VanSchaikTest();
tr.TestRunningLengthWord();
tr.TestSizeInBits1();
tr.TestHasNextSafe();
tr.TestCloneEwahCompressedBitArray();
tr.TestSetGet();
tr.TestWithParameters();
new EWAHCompressedBitArraySerializerTest().TestCustomSerializationStrategy();
}
public void TestMassiveOr()
{
Console.WriteLine("testing massive logical or (can take a couple of minutes)");
int N = 128;
for (int howmany = 512; howmany <= 10000; howmany *= 2)
{
var ewah = new EwahCompressedBitArray[N];
var bset = new BitArray[N];
int k;
for (k = 0; k < ewah.Length; ++k)
{
ewah[k] = new EwahCompressedBitArray();
}
for (k = 0; k < bset.Length; ++k)
{
bset[k] = new BitArray(10000);
}
for (k = 0; k < N; ++k)
{
assertEqualsPositions(bset[k], ewah[k]);
}
for (k = 0; k < howmany; ++k)
{
ewah[(k + 2 * k * k) % ewah.Length].Set(k);
bset[(k + 2 * k * k) % ewah.Length].Set(k, true);
}
for (k = 0; k < N; ++k)
{
assertEqualsPositions(bset[k], ewah[k]);
}
EwahCompressedBitArray answer = ewah[0];
BitArray BitArrayanswer = bset[0];
for (k = 1; k < ewah.Length; ++k)
{
EwahCompressedBitArray tmp = answer.Or(ewah[k]);
BitArrayanswer.Or(bset[k]);
answer = tmp;
assertEqualsPositions(BitArrayanswer, answer);
}
assertEqualsPositions(BitArrayanswer, answer);
k = 0;
foreach (int j in answer)
{
if (k != j)
{
Console.WriteLine(answer.ToDebugString());
}
Assert.AreEqual(k, j);
k += 1;
}
}
Console.WriteLine("testing massive logical or:ok");
}
public void TestYnosa()
{
Console.WriteLine("testing Ynosa");
var a1 = new EwahCompressedBitArray();
var a2 = new EwahCompressedBitArray();
a1.Set(5);
a1.Set(15);
a2.Set(5);
Assert.IsTrue(a1.Intersects(a2));
Console.WriteLine("testing Ynosa:ok");
}
/// <summary>
/// Serializes an instance of <see cref="Ewah.EwahCompressedBitArray"/> into the given stream
/// </summary>
/// <param name="serializationStream">The serialization stream.</param>
/// <param name="bitArray">The bit array.</param>
public void Serialize(Stream serializationStream, EwahCompressedBitArray bitArray)
{
// No actual need to call Shrink with this serialisation strategy, so we can avoid
// mutating the source type (side-effects are bad ;) )
serializationStream.Write( BitConverter.GetBytes(bitArray.SizeInBits), 0, 4 );
serializationStream.Write( BitConverter.GetBytes(bitArray._ActualSizeInWords),0, 4 );
serializationStream.Write(BitConverter.GetBytes(bitArray._Rlw.Position), 0, 4);
for(int i=0; i< bitArray._ActualSizeInWords;i++) {
serializationStream.Write(BitConverter.GetBytes(bitArray._Buffer[i]), 0, 8);
}
return;
}
public void TestCustomSerializationStrategy()
{
Console.WriteLine("testing Custom serialization strategy");
// Create a compressed bit array, and randomly assign up to 20,000 bits to it.
var bmp = new EwahCompressedBitArray();
var r = new Random();
for (int i = 0; i < 23000; i++)
{
if (r.NextDouble() < 0.5)
{
bmp.Set(i);
}
}
byte[] originalDeserialized = null;
byte[] newFormDeserialized = null;
EwahCompressedBitArray newFormReserialized = null;
EwahCompressedBitArray originalReserialized = null;
// First de-serialize+ re-serialize 'normally'
using (var ms = new MemoryStream()) {
BinaryFormatter bf = new BinaryFormatter();
bf.Serialize(ms, bmp);
originalDeserialized = ms.ToArray();
ms.Seek(0, SeekOrigin.Begin);
originalReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
}
// Now de-serialize + re-serialize with the new form.
using (var ms = new MemoryStream()) {
EwahCompressedBitArraySerializer bf = new EwahCompressedBitArraySerializer();
bf.Serialize(ms, bmp);
newFormDeserialized = ms.ToArray();
ms.Seek(0, SeekOrigin.Begin);
newFormReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
}
// Assert that the new form is more compact than the original form.
Assert.Less(newFormDeserialized.Length, originalDeserialized.Length);
// Compare the 'normal' de-serialized + re-serialized form, against the original.
Assert.AreEqual(bmp, originalReserialized);
// Compare the 'new form' de-serialized + re-serialized form, against the original.
Assert.AreEqual(bmp, newFormReserialized);
// Compare the 'normal' de-serialized + re-serialized form, against the newly de-serialized + re-serialized form.
Assert.AreEqual(newFormReserialized, originalReserialized);
Console.WriteLine("testing Custom serialization strategy:ok");
}
/// <summary>
/// Serializes an instance of <see cref="Ewah.EwahCompressedBitArray"/> into the given stream
/// </summary>
/// <param name="serializationStream">The serialization stream.</param>
/// <param name="bitArray">The bit array.</param>
public void Serialize(Stream serializationStream, EwahCompressedBitArray bitArray)
{
// No actual need to call Shrink with this serialisation strategy, so we can avoid
// mutating the source type (side-effects are bad ;-) )
serializationStream.Write(BitConverter.GetBytes(bitArray.SizeInBits), 0, 4);
serializationStream.Write(BitConverter.GetBytes(bitArray._ActualSizeInWords), 0, 4);
serializationStream.Write(BitConverter.GetBytes(bitArray._Rlw.Position), 0, 4);
for (int i = 0; i < bitArray._ActualSizeInWords; i++)
{
serializationStream.Write(BitConverter.GetBytes(bitArray._Buffer[i]), 0, 8);
}
return;
}
public void TestLargeEwahCompressedBitArray()
{
Console.WriteLine("testing EWAH over a large array");
var myarray1 = new EwahCompressedBitArray();
const int n = 11000000;
for (int i = 0; i < n; ++i)
{
myarray1.Set(i);
}
Assert.AreEqual(myarray1.SizeInBits, n);
Console.WriteLine("testing EWAH over a large array:ok");
}
public static void Main(string[] args)
{
var ewahBitmap1 = new EwahCompressedBitArray();
var ewahBitmap2 = new EwahCompressedBitArray();
ewahBitmap1.Set(0);
ewahBitmap1.Set(2);
ewahBitmap1.Set(64);
ewahBitmap1.Set(1 << 30);
Console.WriteLine("Running demo program:");
Console.WriteLine("bitmap 1:");
foreach (int k in ewahBitmap1)
Console.WriteLine(k);
ewahBitmap2.Set(1);
ewahBitmap2.Set(3);
ewahBitmap2.Set(64);
ewahBitmap2.Set(1 << 30);
Console.WriteLine("bitmap 2:");
foreach (int k in ewahBitmap2)
Console.WriteLine(k);
Console.WriteLine();
Console.WriteLine("bitmap 1 OR bitmap 2:");
EwahCompressedBitArray orbitmap = ewahBitmap1.Or(ewahBitmap2);
foreach (int k in orbitmap)
Console.WriteLine(k);
Console.WriteLine("memory usage: " + orbitmap.SizeInBytes + " bytes");
Console.WriteLine();
Console.WriteLine("bitmap 1 AND bitmap 2:");
EwahCompressedBitArray andbitmap = ewahBitmap1.And(ewahBitmap2);
foreach (int k in andbitmap)
Console.WriteLine(k);
Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
Console.WriteLine("bitmap 1 XOR bitmap 2:");
EwahCompressedBitArray xorbitmap = ewahBitmap1.Xor(ewahBitmap2);
foreach (int k in xorbitmap)
Console.WriteLine(k);
Console.WriteLine("memory usage: " + andbitmap.SizeInBytes + " bytes");
Console.WriteLine("End of demo.");
Console.WriteLine("");
var tr = new EwahCompressedBitArrayTest();
tr.TestNot();
tr.TestCardinality();
tr.TestEwahCompressedBitArray();
tr.TestExternalization();
tr.TestLargeEwahCompressedBitArray();
tr.TestMassiveAnd();
tr.TestMassiveAndNot();
tr.TestMassiveOr();
tr.TestMassiveXOR();
tr.HabermaasTest();
tr.VanSchaikTest();
}
public void TestCustomSerializationStrategy()
{
Console.WriteLine("testing Custom serialization strategy");
// Create a compressed bit array, and randomly assign up to 20,000 bits to it.
var bmp = new EwahCompressedBitArray();
var r= new Random();
for (int i = 0; i < 23000; i++) {
if (r.NextDouble() < 0.5) {
bmp.Set(i);
}
}
byte[] originalDeserialized= null;
byte[] newFormDeserialized= null;
EwahCompressedBitArray newFormReserialized= null;
EwahCompressedBitArray originalReserialized= null;
// First de-serialize+ re-serialize 'normally'
using (var ms = new MemoryStream()) {
BinaryFormatter bf = new BinaryFormatter();
bf.Serialize(ms, bmp);
originalDeserialized = ms.ToArray();
ms.Seek(0, SeekOrigin.Begin);
originalReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
}
// Now de-serialize + re-serialize with the new form.
using (var ms = new MemoryStream()) {
EwahCompressedBitArraySerializer bf = new EwahCompressedBitArraySerializer();
bf.Serialize(ms, bmp);
newFormDeserialized = ms.ToArray();
ms.Seek(0, SeekOrigin.Begin);
newFormReserialized = (EwahCompressedBitArray)bf.Deserialize(ms);
}
// Assert that the new form is more compact than the original form.
Assert.Less(newFormDeserialized.Length, originalDeserialized.Length);
// Compare the 'normal' de-serialized + re-serialized form, against the original.
Assert.AreEqual(bmp, originalReserialized);
// Compare the 'new form' de-serialized + re-serialized form, against the original.
Assert.AreEqual(bmp, newFormReserialized);
// Compare the 'normal' de-serialized + re-serialized form, against the newly de-serialized + re-serialized form.
Assert.AreEqual(newFormReserialized, originalReserialized);
Console.WriteLine("testing Custom serialization strategy:ok");
}
public void TestIntersectOddNess()
{
Console.WriteLine("testing IntersectOddNess");
var a1 = new EwahCompressedBitArray();
var a2 = new EwahCompressedBitArray();
a1.Set(12);
a2.Set(0);
a2.Set(1);
a2.Set(4);
a2.Set(14);
Assert.IsFalse(a1.Intersects(a2));
Console.WriteLine("testing IntersectOddNess:ok");
}
public void TestNot()
{
Console.WriteLine("testing not");
var bmp = new EwahCompressedBitArray();
for (int i = 0; i <= 184; i++)
{
bmp.Set(i);
}
Assert.AreEqual(185, bmp.GetCardinality());
bmp.Not();
Assert.AreEqual(0, bmp.GetCardinality());
Console.WriteLine("testing not:ok");
}
/**
* Convenience function to assess equality between a compressed BitArray
* and an uncompressed BitArray
*
* @param x the compressed BitArray/bitmap
* @param y the uncompressed BitArray/bitmap
*/
private static void AreEqual(EwahCompressedBitArray x, BitArray y)
{
Assert.AreEqual(x.GetCardinality(), y.Cardinality());
var positions = new List <int>();
for (int ii = 0; ii < y.Count; ii++)
{
if (y[ii])
{
positions.Add(ii);
}
}
AreEqual(x.GetPositions(), positions);
}
public UniversalSubjectPosition2(SortedSet <long> allRITEIds)
{
CausesOfLoss = new HashSet <SymbolicValue>()
{
"EQ", "WS", "CS", "FL", "WT", "FR", "TR", "SH", "FF", "SL", "WI", "SU", "HA", "TO", "SW", "FZ", "IC", "SN", "WF", "TS"
};
ResolvedExposureTypes = new HashSet <int>(ExposureType.GetIndividualIntExposureTypes(ExposureType.EExposureType.Loss));
AllRITEIds = new EwahCompressedBitArray();
OffsetForAllRITEIds = allRITEIds.First();
foreach (long RITEId in allRITEIds)
{
AllRITEIds.Set((int)(RITEId - OffsetForAllRITEIds));
}
}
public void TestSetGet()
{
Console.WriteLine("testing EWAH Set/get");
var ewcb = new EwahCompressedBitArray();
int[] val = { 5, 4400, 44600, 55400, 1000000 };
for (int k = 0; k < val.Length; ++k)
{
ewcb.Set(val[k]);
}
List <int> result = ewcb.GetPositions();
AreEqual(val, result);
Console.WriteLine("testing EWAH Set/get:ok");
}
//
/**
* Assess equality between an uncompressed bitmap and a compressed one,
* part of a test contributed by Marc Polizzi
*
* @param clrBitArray the clr BitArray
* @param ewahBitmap the ewah BitArray
*/
private static void assertEqualsIterator(BitArray clrBitArray, EwahCompressedBitArray ewahBitmap)
{
var positions = new List <int>();
foreach (int bit in ewahBitmap)
{
Assert.IsTrue(clrBitArray.Get(bit), "enumerator: BitArray got different bits");
positions.Add(bit);
}
for (int pos = clrBitArray.NextSetBit(0); pos >= 0; pos = clrBitArray.NextSetBit(pos + 1))
{
Assert.IsTrue(positions.Contains(pos), "enumerator: BitArray got different bits");
}
}
/**
* Pseudo-non-deterministic test inspired by S.J.vanSchaik.
* (Yes, non-deterministic tests are bad, but the test is actually deterministic.)
*/
/**
* Pseudo-non-deterministic test inspired by Federico Fissore.
*
* @param length the number of set bits in a bitmap
*/
private static void ShouldSetBits(int length)
{
Console.WriteLine("testing shouldSetBits " + length);
int[] bitsToSet = CreateSortedIntArrayOfBitsToSet(length, 434222);
var ewah = new EwahCompressedBitArray();
Console.WriteLine(" ... setting " + bitsToSet.Length + " values");
foreach (int i in bitsToSet)
{
ewah.Set(i);
}
Console.WriteLine(" ... verifying " + bitsToSet.Length + " values");
AreEqual(ewah, bitsToSet);
Console.WriteLine(" ... checking GetCardinality");
Assert.AreEqual(bitsToSet.Length, ewah.GetCardinality());
}
public void TayaraTest()
{
Console.WriteLine("Tayara test");
for (int offset = 64; offset < (1 << 30); offset *= 2)
{
EwahCompressedBitArray a = new EwahCompressedBitArray();
EwahCompressedBitArray b = new EwahCompressedBitArray();
for (int k = 0; k < 64; ++k)
{
a.Set(offset + k);
b.Set(offset + k);
}
Assert.AreEqual(a.And(b).Equals(a), true);
Assert.AreEqual(a.Or(b).Equals(a), true);
}
}
public void TestCloneEwahCompressedBitArray()
{
Console.WriteLine("testing EWAH clone");
EwahCompressedBitArray a = new EwahCompressedBitArray();
a.Set(410018);
a.Set(410019);
a.Set(410020);
a.Set(410021);
a.Set(410022);
a.Set(410023);
EwahCompressedBitArray b = (EwahCompressedBitArray)a.Clone();
a.SetSizeInBits(487123, false);
b.SetSizeInBits(487123, false);
Assert.AreEqual(a, b);
}
public void TestMassiveXOR()
{
Console.WriteLine("testing massive xor (can take a couple of minutes)");
int N = 16;
var ewah = new EwahCompressedBitArray[N];
var bset = new BitArray[N];
for (int k = 0; k < ewah.Length; ++k)
{
ewah[k] = new EwahCompressedBitArray();
}
for (int k = 0; k < bset.Length; ++k)
{
bset[k] = new BitArray(30000);
}
for (int k = 0; k < 30000; ++k)
{
ewah[(k + 2 * k * k) % ewah.Length].Set(k);
bset[(k + 2 * k * k) % ewah.Length].Set(k, true);
}
EwahCompressedBitArray answer = ewah[0];
BitArray BitArrayanswer = bset[0];
for (int k = 1; k < ewah.Length; ++k)
{
answer = answer.Xor(ewah[k]);
BitArrayanswer.Xor(bset[k]);
assertEqualsPositions(BitArrayanswer, answer);
}
int k2 = 0;
foreach (int j in answer)
{
if (k2 != j)
{
Console.WriteLine(answer.ToDebugString());
}
Assert.AreEqual(k2, j);
k2 += 1;
}
Console.WriteLine("testing massive xor:ok");
}
public void SsiYanKaiTest()
{
Console.WriteLine("testing SsiYanKaiTest");
EwahCompressedBitArray a = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008, 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017, 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026, 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035, 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044, 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053, 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062, 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, 40071, 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100);
EwahCompressedBitArray b = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 270000);
HashSet <int> aPositions = new HashSet <int>(a.GetPositions());
int intersection = 0;
EwahCompressedBitArray inter = new EwahCompressedBitArray();
HashSet <int> bPositions = new HashSet <int>(b.GetPositions());
foreach (int integer in bPositions)
{
if (aPositions.Contains(integer))
{
inter.Set(integer);
++intersection;
}
}
EwahCompressedBitArray and2 = a.And(b);
List <int> l1 = inter.GetPositions();
List <int> l2 = and2.GetPositions();
var ok = true;
if (l1.Count != l2.Count)
{
Console.WriteLine("cardinality differs = " + l1.Count + " " + l2.Count);
ok = false;
}
for (int k = 0; k < l1.Count; ++k)
{
if (l1[k] != l2[k])
{
Console.WriteLine("differ at " + k + " = " + l1[k] + " " + l2[k]);
ok = false;
}
}
Assert.IsTrue(ok);
Assert.AreEqual(true, and2.Equals(inter));
Assert.AreEqual(inter.GetHashCode(), and2.GetHashCode());
Assert.AreEqual(intersection, and2.GetCardinality());
}
// part of a test contributed by Marc Polizzi
/**
* Assert equals positions.
*
* @param clrBitArray the jdk bitmap
* @param ewahBitmap the ewah bitmap
*/
private static void assertEqualsPositions(BitArray clrBitArray,
EwahCompressedBitArray ewahBitmap)
{
List <int> positions = ewahBitmap.GetPositions();
foreach (int position in positions)
{
Assert.IsTrue(clrBitArray.Get(position),
"positions: BitArray got different bits");
}
var ps = new HashSet <int>(positions);
for (int pos = clrBitArray.NextSetBit(0);
pos >= 0;
pos = clrBitArray
.NextSetBit(pos + 1))
{
Assert.IsTrue(ps.Contains(pos),
"positions: BitArray got different bits");
}
}
private EwahCompressedBitArray getCopyBitmap(EwahCompressedBitArray bitmap)
{
var bit = (EwahCompressedBitArray)bitmap.Clone();
bit.SetSizeInBits(_maxBitSize, false);
return bit;
}
public EwahCompressedBitArray query(BICriteria criteria)
{
// This method could use recursion which would make it more readable
// For performance reasons, and to avoid StackOverflowException, it
// uses the Snapshot class to avoid recursion
if (criteria == null)
throw new ArgumentNullException("criteria");
EwahCompressedBitArray temp;
Snapshot previous;
Snapshot next;
Snapshot current = new Snapshot();
current.criteria = criteria;
current.state = 0;
Stack<Snapshot> stack = new Stack<Snapshot>();
stack.Push(current);
while (stack.Count > 0)
{
current = stack.Pop();
if (stack.Count > 0)
previous = stack.Peek();
else
previous = null;
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.OR ||
current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
{
if (current.state == 0)
{
current.state = 1;
stack.Push(current);
next = new Snapshot();
next.criteria = current.criteria.LeftCriteria;
next.state = 0;
stack.Push(next);
continue;
}
else if (current.state == 1)
{
current.state = 2;
stack.Push(current);
next = new Snapshot();
next.criteria = current.criteria.RightCriteria;
next.state = 0;
stack.Push(next);
continue;
}
else
{
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.AND)
temp = (current.left.And(current.right));
else
temp = (current.left.Or(current.right));
if (previous == null)
{
return temp;
}
else if (previous.state == 1)
{
previous.left = temp;
}
else if (previous.state == 2)
{
previous.right = temp;
}
continue;
}
}
else
{
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EMPTY_ONLY)
{
if (previous == null)
return getEmptyBitmap(current.criteria.Key);
else if (previous.state == 1)
previous.left = getEmptyBitmap(current.criteria.Key);
else if (previous.state == 2)
previous.right = getEmptyBitmap(current.criteria.Key);
continue;
}
else
{
EwahCompressedBitArray bitmap;
if (!_bitmaps.ContainsKey(current.criteria.Key))
bitmap = getFilledBitmap(false);
else
bitmap = getCopyBitmap(_bitmaps[current.criteria.Key]);
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS ||
current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY)
bitmap.Not();
if (current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.NOT_EQUALS_OR_EMPTY ||
current.criteria.CriteriaOperator == BusinessLayer.Uteis.BICriteria.Operator.EQUALS_OR_EMPTY)
bitmap = bitmap.Or(getEmptyBitmap(current.criteria.Key));
if (previous == null)
return bitmap;
else if (previous.state == 1)
previous.left = bitmap;
else if (previous.state == 2)
previous.right = bitmap;
continue;
}
}
}
temp = new EwahCompressedBitArray();
temp.SetSizeInBits(_maxBitSize, false);
return temp;
}
public void Set(BIKey key, int bit)
{
EwahCompressedBitArray bitmap;
EwahCompressedBitArray emptyBitmap;
if (_bitmaps.ContainsKey(key))
bitmap = _bitmaps[key];
else
{
bitmap = new EwahCompressedBitArray();
_bitmaps.Add(key, bitmap);
}
if (_emptyBitmaps.ContainsKey(key.Group))
emptyBitmap = _emptyBitmaps[key.Group];
else
{
emptyBitmap = new EwahCompressedBitArray();
_emptyBitmaps.Add(key.Group, emptyBitmap);
}
bitmap.Set(bit);
emptyBitmap.Not();
emptyBitmap.Set(bit);
emptyBitmap.Not();
_maxBitSize = (_maxBitSize < (bit + 1) ? (bit + 1) : _maxBitSize);
}
private EwahCompressedBitArray getFilledBitmap(bool fill)
{
EwahCompressedBitArray bitmap = new EwahCompressedBitArray();
bitmap.SetSizeInBits(_maxBitSize, fill);
return bitmap;
}