internal EwahCompressedBitArray CreateBitMapIndexForExcludedTags(CLR.HashSet <string> tagsToExclude, QueryType queryType, bool printLoggingMessages = false)
{
var bitMapTimer = Stopwatch.StartNew();
var tagLookupForQueryType = GetTagByQueryLookup(queryType);
var collectIdsTimer = Stopwatch.StartNew();
var excludedQuestionIds = cache.Value.GetCachedHashSet();
foreach (var tag in tagsToExclude)
{
foreach (var id in tagLookupForQueryType[tag])
{
excludedQuestionIds.Add(id);
}
}
collectIdsTimer.Stop();
// At the end we need to have the BitMap Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded
// That way we can efficiently apply the exclusions by ANDing this BitMap to the previous results
var allQuestions = tagLookupForQueryType[TagServer.ALL_TAGS_KEY];
var setBitsTimer = Stopwatch.StartNew();
var bitMap = new EwahCompressedBitArray();
for (int index = 0; index < allQuestions.Length; index++)
{
if (excludedQuestionIds.Contains(allQuestions[index]))
{
var wasSet = bitMap.SetOptimised(index); // Set a bit where you CAN'T use a question
if (wasSet == false)
{
Logger.LogStartupMessage("Error, unable to set bit {0:N0} (SizeInBits = {1:N0})", index, bitMap.SizeInBits);
}
}
}
setBitsTimer.Stop();
var tidyUpTimer = Stopwatch.StartNew();
bitMap.SetSizeInBits(questions.Count, defaultvalue: false);
bitMap.Shrink();
tidyUpTimer.Stop();
bitMapTimer.Stop();
if (printLoggingMessages)
{
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to collect {2:N0} Question Ids from {3:N0} Tags",
collectIdsTimer.Elapsed, collectIdsTimer.ElapsedMilliseconds, excludedQuestionIds.Count, tagsToExclude.Count);
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to set {2:N0} bits",
setBitsTimer.Elapsed, setBitsTimer.ElapsedMilliseconds, bitMap.GetCardinality());
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) to tidy-up the Bit Map (SetSizeInBits(..) and Shrink()), Size={2:N0} bytes ({3:N2} MB)",
tidyUpTimer.Elapsed, tidyUpTimer.ElapsedMilliseconds, bitMap.SizeInBytes, bitMap.SizeInBytes / 1024.0 / 1024.0);
using (Utils.SetConsoleColour(ConsoleColor.DarkYellow))
{
Logger.LogStartupMessage("Took {0} ({1,6:N0} ms) in TOTAL, made BitMap from {2:N0} Tags ({3:N0} Qu Ids), Cardinality={4:N0} ({5:N0})\n",
bitMapTimer.Elapsed, bitMapTimer.ElapsedMilliseconds,
tagsToExclude.Count,
excludedQuestionIds.Count,
bitMap.GetCardinality(),
(ulong)questions.Count - bitMap.GetCardinality());
}
}
return(bitMap);
}