public void TestMassiveAndNot()
{
Console.WriteLine("testing massive and not");
int N = 1024;
var ewah = new EwahCompressedBitArray[N];
for (int k = 0; k < ewah.Length; ++k)
{
ewah[k] = new EwahCompressedBitArray();
}
for (int k = 0; k < 30000; ++k)
{
ewah[(k + 2 * k * k) % ewah.Length].Set(k);
}
EwahCompressedBitArray answer = ewah[0];
EwahCompressedBitArray answer2 = ewah[0];
;
for (int k = 1; k < ewah.Length; ++k)
{
answer = answer.AndNot(ewah[k]);
EwahCompressedBitArray copy = null;
try
{
copy = (EwahCompressedBitArray)ewah[k].Clone();
copy.Not();
answer2.And(copy);
assertEqualsPositions(answer.GetPositions(), answer2.GetPositions());
}
catch (InvalidOperationException e)
{
Console.Error.WriteLine(e.StackTrace);
}
}
Console.WriteLine("testing massive and not:ok");
}
internal QueryResult Query(QueryInfo info, EwahCompressedBitArray exclusionBitMap = null, bool printLoggingMessages = false)
{
var bitMap = GetTagByQueryBitMapLookup(info.Type);
var questionLookup = GetTagByQueryLookup(info.Type)[TagServer.ALL_TAGS_KEY];
// Calculating the Cardinality can be (is?) expensive, we don't want to do it in Queries unless we really need to!?
bool calculateCardinality = true; // false
if (printLoggingMessages)
{
Logger.Log("Tag \"{0}\" is in {1:N0} Questions, Tag \"{2}\" is in {3:N0} Questions",
info.Tag, allTags[info.Tag], info.OtherTag, allTags[info.OtherTag]);
}
//PrintResults(Enumerable.Range(0, questionLookup.Length), qu => questionLookup[qu], TagServer.ALL_TAGS_KEY, info.Type);
//PrintResults(bitMap[info.Tag], qu => questionLookup[qu], info.Tag, info.Type);
//PrintResults(bitMap[info.OtherTag], qu => questionLookup[qu], info.OtherTag, info.Type);
var timer = Stopwatch.StartNew();
var tag1BitMap = bitMap[info.Tag];
var tag2BitMap = bitMap[info.OtherTag];
EwahCompressedBitArray bitMapResult = new EwahCompressedBitArray();
switch (info.Operator)
{
case "AND":
bitMapResult = tag1BitMap.And(tag2BitMap);
break;
case "AND-NOT":
bitMapResult = tag1BitMap.AndNot(tag2BitMap);
break;
case "OR":
bitMapResult = tag1BitMap.Or(tag2BitMap);
break;
case "OR-NOT": //"i.e. .net+or+jquery-"
bitMapResult = tag1BitMap.OrNot(tag2BitMap);
break;
// TODO Work out what a "NOT" query really means, the LINQ version was "result = tag1Query.Except(tag2Query)" (which is the same as AND-NOT?!)
//case "NOT":
// var bitMapResult = (EwahCompressedBitArray)tag2BitMap.Clone();
// bitMapResult.Not();
// break;
default:
throw new InvalidOperationException(string.Format("Invalid operator specified: {0}", info.Operator ?? "<NULL>"));
}
if (exclusionBitMap != null)
{
ulong cardinalityBeforeExclusions = 0;
if (printLoggingMessages)
{
cardinalityBeforeExclusions = bitMapResult.GetCardinality();
}
// The Exclusiong BitMap is Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
// That way we can efficiently apply the exclusions by ANDing this BitMap to the previous results
var exclusionTimer = Stopwatch.StartNew();
bitMapResult = bitMapResult.AndNot(exclusionBitMap);
exclusionTimer.Stop();
if (printLoggingMessages)
{
if (calculateCardinality)
{
Logger.Log("Took {0,5:N2} ms to apply exclusion BitMap (Cardinality={1:N0}), Results Cardinality: Before={2:N0}, After={3:N0}",
exclusionTimer.Elapsed.TotalMilliseconds, exclusionBitMap.GetCardinality(), cardinalityBeforeExclusions, bitMapResult.GetCardinality());
}
else
{
Logger.Log("Took {0,5:N2} ms to apply exclusion BitMap", exclusionTimer.Elapsed.TotalMilliseconds);
}
}
}
var resultCollectionTimer = Stopwatch.StartNew();
var result = bitMapResult.Skip(info.Skip)
.Take(info.PageSize)
.Select(i => questions[questionLookup[i]])
.ToList();
resultCollectionTimer.Stop();
if (printLoggingMessages)
{
Logger.Log("Took {0,5:N2} ms to collect the results", resultCollectionTimer.Elapsed.TotalMilliseconds);
}
timer.Stop();
Results.AddData(timer.Elapsed.TotalMilliseconds.ToString("#.##"));
if (printLoggingMessages)
{
using (Utils.SetConsoleColour(ConsoleColor.DarkYellow))
{
if (calculateCardinality)
{
Logger.Log("Took {0,5:N2} ms in TOTAL to calculate \"{1} {2} {3}\", Got {4} results, (Result Cardinality={5:N0})",
timer.Elapsed.TotalMilliseconds, info.Tag, info.Operator, info.OtherTag, result.Count, bitMapResult.GetCardinality());
}
else
{
Logger.Log("Took {0,5:N2} ms in TOTAL to calculate \"{1} {2} {3}\", Got {4} results",
timer.Elapsed.TotalMilliseconds, info.Tag, info.Operator, info.OtherTag, result.Count);
}
}
//PrintResults(bitMapResult, qu => questionLookup[qu], string.Format("{0} {1} {2}", info.Tag, info.Operator, info.OtherTag), info.Type);
Logger.Log();
}
return(new QueryResult
{
Questions = result,
// TODO see if we can get meaningful numbers here, WITHOUT calling GetCardinality() (because it's expensive)
//Counters = new Dictionary<string, int>
//{
// { "TagCounter", tagCounter },
// { "OtherTagCounter", otherTagCounter },
// { "ExclusionCounter", exclusionCounter.Counter }
//}
});
}
/**
* a non-deterministic test proposed by Marc Polizzi.
*
* @param maxlength the maximum uncompressed size of the bitmap
*/
public static void PolizziTest(int maxlength)
{
Console.WriteLine("Polizzi test with max length = " + maxlength);
for (int k = 0; k < 10000; k += 77)
{
var rnd = new Random();
var ewahBitmap1 = new EwahCompressedBitArray();
var clrBitArray1 = new BitArray(10000);
var ewahBitmap2 = new EwahCompressedBitArray();
var clrBitArray2 = new BitArray(10000);
int len = rnd.Next(maxlength);
for (int pos = 0; pos < len; pos++)
{
// random *** number of bits set ***
if (rnd.Next(7) == 0)
{
// random *** increasing *** values
ewahBitmap1.Set(pos);
clrBitArray1.Set(pos, true);
}
if (rnd.Next(11) == 0)
{
// random *** increasing *** values
ewahBitmap2.Set(pos);
clrBitArray2.Set(pos, true);
}
}
assertEquals(clrBitArray1, ewahBitmap1);
assertEquals(clrBitArray2, ewahBitmap2);
// XOR
{
EwahCompressedBitArray xorEwahBitmap = ewahBitmap1.Xor(ewahBitmap2);
var xorclrBitArray = (BitArray)clrBitArray1.Clone();
xorclrBitArray.Xor(clrBitArray2);
assertEquals(xorclrBitArray, xorEwahBitmap);
}
// AND
{
EwahCompressedBitArray andEwahBitmap = ewahBitmap1.And(ewahBitmap2);
var andclrBitArray = (BitArray)clrBitArray1.Clone();
andclrBitArray.And(clrBitArray2);
assertEquals(andclrBitArray, andEwahBitmap);
}
// AND
{
EwahCompressedBitArray andEwahBitmap = ewahBitmap2.And(ewahBitmap1);
var andclrBitArray = (BitArray)clrBitArray1.Clone();
andclrBitArray.And(clrBitArray2);
assertEquals(andclrBitArray, andEwahBitmap);
}
// AND NOT
{
EwahCompressedBitArray andNotEwahBitmap = ewahBitmap1
.AndNot(ewahBitmap2);
var andNotclrBitArray = (BitArray)clrBitArray1.Clone();
andNotclrBitArray.AndNot(clrBitArray2);
assertEquals(andNotclrBitArray, andNotEwahBitmap);
}
// AND NOT
{
EwahCompressedBitArray andNotEwahBitmap = ewahBitmap2
.AndNot(ewahBitmap1);
var andNotclrBitArray = (BitArray)clrBitArray2.Clone();
andNotclrBitArray.AndNot(clrBitArray1);
assertEquals(andNotclrBitArray, andNotEwahBitmap);
}
// OR
{
EwahCompressedBitArray orEwahBitmap = ewahBitmap1.Or(ewahBitmap2);
var orclrBitArray = (BitArray)clrBitArray1.Clone();
orclrBitArray.Or(clrBitArray2);
assertEquals(orclrBitArray, orEwahBitmap);
}
// OR
{
EwahCompressedBitArray orEwahBitmap = ewahBitmap2.Or(ewahBitmap1);
var orclrBitArray = (BitArray)clrBitArray1.Clone();
orclrBitArray.Or(clrBitArray2);
assertEquals(orclrBitArray, orEwahBitmap);
}
}
}
private static EwahCompressedBitArray Minus(this EwahCompressedBitArray FromThis, EwahCompressedBitArray SubtractThis)
{
return(FromThis.AndNot(SubtractThis));
}