public void TestMassiveAnd()
{
Console.WriteLine("testing massive logical and");
var ewah = new EwahCompressedBitArray[1024];
for (int k = 0; k < ewah.Length; ++k)
{
ewah[k] = new EwahCompressedBitArray();
}
for (int k = 0; k < 30000; ++k)
{
ewah[(k + 2 * k * k) % ewah.Length].Set(k);
}
EwahCompressedBitArray answer = ewah[0];
for (int k = 1; k < ewah.Length; ++k)
{
answer = answer.And(ewah[k]);
}
// result should be empty
if (answer.GetPositions().Count != 0)
{
Console.WriteLine(answer.ToDebugString());
}
Assert.IsTrue(answer.GetPositions().Count == 0);
Console.WriteLine("testing massive logical and:ok");
}
public void EwahIteratorProblem()
{
Console.WriteLine("testing ArnonMoscona");
var bitmap = new EwahCompressedBitArray();
for (int i = 9434560; i <= 9435159; i++)
{
bitmap.Set(i);
}
List <int> v = bitmap.GetPositions();
int k = 0;
foreach (int ival in bitmap)
{
Assert.AreEqual(ival, v[k++]);
}
Assert.AreEqual(k, v.Count);
for (k = 2; k <= 1024; k *= 2)
{
int[] bitsToSet = CreateSortedIntArrayOfBitsToSet(k, 434455 + 5 * k);
var ewah = new EwahCompressedBitArray();
foreach (int i in bitsToSet)
{
ewah.Set(i);
}
assertEqualsPositions(bitsToSet, ewah.GetPositions());
}
}
internal void ValidateExclusionBitMap(EwahCompressedBitArray bitMapIndex, CLR.HashSet <string> expandedTagsNGrams, QueryType queryType)
{
// Exclusion BitMap is Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
var questionLookup = GetTagByQueryLookup(queryType)[TagServer.ALL_TAGS_KEY];
var invalidQuestions = new List <Tuple <Question, string> >();
var NOTbitMapIndex = ((EwahCompressedBitArray)bitMapIndex.Clone());
NOTbitMapIndex.Not();
var positions = NOTbitMapIndex.GetPositions();
foreach (var position in positions)
{
var question = questions[questionLookup[position]];
foreach (var tag in question.Tags)
{
if (expandedTagsNGrams.Contains(tag))
{
invalidQuestions.Add(Tuple.Create(question, tag));
}
}
// Sometimes the validitation locks up my laptop, this *seems* to make a difference?!
Thread.Yield();
}
using (Utils.SetConsoleColour(ConsoleColor.Blue))
Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for INVALID tags", positions.Count);
if (invalidQuestions.Any())
{
using (Utils.SetConsoleColour(ConsoleColor.Red))
Logger.Log("ERROR Validating Exclusion Bit Map, {0:N0} questions should have been excluded",
invalidQuestions.Select(i => i.Item1.Id).Distinct().Count());
foreach (var error in invalidQuestions)
{
Logger.Log(" {0,8}: {1} -> {2}", error.Item1.Id, String.Join(", ", error.Item1.Tags), error.Item2);
}
}
var expectedPositions = bitMapIndex.GetPositions();
foreach (var position in expectedPositions)
{
var question = questions[questionLookup[position]];
if (question.Tags.Any(t => expandedTagsNGrams.Contains(t)) == false)
{
using (Utils.SetConsoleColour(ConsoleColor.Red))
Logger.Log("ERROR {0,8}: {1} -> didn't contain ANY excluded tags", question.Id, String.Join(", ", question.Tags));
}
}
using (Utils.SetConsoleColour(ConsoleColor.Blue))
Logger.Log("Validating Exclusion Bit Map, checked {0:N0} positions for EXPECTED tags", expectedPositions.Count);
Logger.Log();
}
/**
* Convenience function to assess equality between a compressed BitArray
* and an uncompressed BitArray
*
* @param x the compressed BitArray/bitmap
* @param y the uncompressed BitArray/bitmap
*/
private static void AreEqual(EwahCompressedBitArray x, BitArray y)
{
Assert.AreEqual(x.GetCardinality(), y.Cardinality());
var positions = new List <int>();
for (int ii = 0; ii < y.Count; ii++)
{
if (y[ii])
{
positions.Add(ii);
}
}
AreEqual(x.GetPositions(), positions);
}
public void TestSetGet()
{
Console.WriteLine("testing EWAH Set/get");
var ewcb = new EwahCompressedBitArray();
int[] val = { 5, 4400, 44600, 55400, 1000000 };
for (int k = 0; k < val.Length; ++k)
{
ewcb.Set(val[k]);
}
List <int> result = ewcb.GetPositions();
AreEqual(val, result);
Console.WriteLine("testing EWAH Set/get:ok");
}
public void SsiYanKaiTest()
{
Console.WriteLine("testing SsiYanKaiTest");
EwahCompressedBitArray a = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008, 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017, 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026, 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035, 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044, 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053, 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062, 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, 40071, 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100);
EwahCompressedBitArray b = EwahCompressedBitArray.BitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 270000);
HashSet <int> aPositions = new HashSet <int>(a.GetPositions());
int intersection = 0;
EwahCompressedBitArray inter = new EwahCompressedBitArray();
HashSet <int> bPositions = new HashSet <int>(b.GetPositions());
foreach (int integer in bPositions)
{
if (aPositions.Contains(integer))
{
inter.Set(integer);
++intersection;
}
}
EwahCompressedBitArray and2 = a.And(b);
List <int> l1 = inter.GetPositions();
List <int> l2 = and2.GetPositions();
var ok = true;
if (l1.Count != l2.Count)
{
Console.WriteLine("cardinality differs = " + l1.Count + " " + l2.Count);
ok = false;
}
for (int k = 0; k < l1.Count; ++k)
{
if (l1[k] != l2[k])
{
Console.WriteLine("differ at " + k + " = " + l1[k] + " " + l2[k]);
ok = false;
}
}
Assert.IsTrue(ok);
Assert.AreEqual(true, and2.Equals(inter));
Assert.AreEqual(inter.GetHashCode(), and2.GetHashCode());
Assert.AreEqual(intersection, and2.GetCardinality());
}
// part of a test contributed by Marc Polizzi
/**
* Assert equals positions.
*
* @param clrBitArray the jdk bitmap
* @param ewahBitmap the ewah bitmap
*/
private static void assertEqualsPositions(BitArray clrBitArray,
EwahCompressedBitArray ewahBitmap)
{
List <int> positions = ewahBitmap.GetPositions();
foreach (int position in positions)
{
Assert.IsTrue(clrBitArray.Get(position),
"positions: BitArray got different bits");
}
var ps = new HashSet <int>(positions);
for (int pos = clrBitArray.NextSetBit(0);
pos >= 0;
pos = clrBitArray
.NextSetBit(pos + 1))
{
Assert.IsTrue(ps.Contains(pos),
"positions: BitArray got different bits");
}
}
public void testDebugSetSizeInBitsTest()
{
Console.WriteLine("testing DebugSetSizeInBits");
EwahCompressedBitArray b = new EwahCompressedBitArray();
b.Set(4);
b.SetSizeInBits(6, true);
List <int> positions = b.GetPositions();
Assert.AreEqual(2, positions.Count);
Assert.AreEqual(4, positions[0]);
Assert.AreEqual(5, positions[1]);
IEnumerator <int> iterator = ((IEnumerable <int>)b).GetEnumerator();
Assert.AreEqual(true, iterator.MoveNext());
Assert.AreEqual(4, iterator.Current);
Assert.AreEqual(true, iterator.MoveNext());
Assert.AreEqual(5, iterator.Current);
Assert.AreEqual(false, iterator.MoveNext());
}
public void TestMassiveAndNot()
{
Console.WriteLine("testing massive and not");
int N = 1024;
var ewah = new EwahCompressedBitArray[N];
for (int k = 0; k < ewah.Length; ++k)
{
ewah[k] = new EwahCompressedBitArray();
}
for (int k = 0; k < 30000; ++k)
{
ewah[(k + 2 * k * k) % ewah.Length].Set(k);
}
EwahCompressedBitArray answer = ewah[0];
EwahCompressedBitArray answer2 = ewah[0];
;
for (int k = 1; k < ewah.Length; ++k)
{
answer = answer.AndNot(ewah[k]);
EwahCompressedBitArray copy = null;
try
{
copy = (EwahCompressedBitArray)ewah[k].Clone();
copy.Not();
answer2.And(copy);
assertEqualsPositions(answer.GetPositions(), answer2.GetPositions());
}
catch (InvalidOperationException e)
{
Console.Error.WriteLine(e.StackTrace);
}
}
Console.WriteLine("testing massive and not:ok");
}
public void TestExternalization()
{
Console.WriteLine("testing EWAH externalization");
var ewcb = new EwahCompressedBitArray();
int[] val = { 5, 4400, 44600, 55400, 1000000 };
foreach (int t in val)
{
ewcb.Set(t);
}
var bos = new MemoryStream();
var bf = new BinaryFormatter();
bf.Serialize(bos, ewcb);
bos.Position = 0;
ewcb = (EwahCompressedBitArray)bf.Deserialize(bos);
List <int> result = ewcb.GetPositions();
AreEqual(val, result);
Console.WriteLine("testing EWAH externalization:ok");
}
public void TestEwahCompressedBitArray()
{
Console.WriteLine("testing EWAH (basic)");
const long zero = 0;
const long specialval = 1L | (1L << 4) | (1L << 63);
const long notzero = ~zero;
var myarray1 = new EwahCompressedBitArray
{
zero, zero, zero, specialval, specialval, notzero, zero
};
Assert.AreEqual(myarray1.GetPositions().Count, 6 + 64);
var myarray2 = new EwahCompressedBitArray();
myarray2.Add(zero);
myarray2.Add(specialval);
myarray2.Add(specialval);
myarray2.Add(notzero);
myarray2.Add(zero);
myarray2.Add(zero);
myarray2.Add(zero);
Assert.AreEqual(myarray2.GetPositions().Count, 6 + 64);
List <int> data1 = myarray1.GetPositions();
List <int> data2 = myarray2.GetPositions();
var logicalor = new List <int>();
{
var tmp = new HashSet <int>();
tmp.AddRange(data1);
tmp.AddRange(data2);
logicalor.AddRange(tmp);
}
logicalor.Sort();
var logicaland = new List <int>();
logicaland.AddRange(data1);
logicaland.Retain(data2);
logicaland.Sort();
EwahCompressedBitArray arrayand = myarray1.And(myarray2);
AreEqual(arrayand.GetPositions(), logicaland);
EwahCompressedBitArray arrayor = myarray1.Or(myarray2);
AreEqual(arrayor.GetPositions(), logicalor);
EwahCompressedBitArray arrayandbis = myarray2.And(myarray1);
AreEqual(arrayandbis.GetPositions(), logicaland);
EwahCompressedBitArray arrayorbis = myarray2.Or(myarray1);
AreEqual(arrayorbis.GetPositions(), logicalor);
var x = new EwahCompressedBitArray();
foreach (int i in myarray1.GetPositions())
{
x.Set(i);
}
AreEqual(x.GetPositions(), myarray1.GetPositions());
x = new EwahCompressedBitArray();
foreach (int i in myarray2.GetPositions())
{
x.Set(i);
}
AreEqual(x.GetPositions(), myarray2.GetPositions());
x = new EwahCompressedBitArray();
foreach (int pos in myarray1)
{
x.Set(pos);
}
AreEqual(x.GetPositions(), myarray1.GetPositions());
x = new EwahCompressedBitArray();
foreach (int pos in myarray2)
{
x.Set(pos);
}
AreEqual(x.GetPositions(), myarray2.GetPositions());
Console.WriteLine("testing EWAH (basic):ok");
}