internal QueryResult Query(QueryInfo info, CLR.HashSet <string> tagsToExclude = null) { var timer = Stopwatch.StartNew(); TagByQueryLookup queryInfo = GetTagByQueryLookup(info.Type); ThrowIfInvalidParameters(info.Tag, info.PageSize, queryInfo); ThrowIfInvalidParameters(info.OtherTag, info.PageSize, queryInfo); var tagCounter = 0; var otherTagCounter = 0; var exclusionCounter = new CounterWrapper(initialValue: 0); IEnumerable <int> tag1Query = queryInfo[info.Tag].Select(t => { tagCounter++; return(t); }); IEnumerable <int> tag2Query = queryInfo[info.OtherTag].Select(t => { otherTagCounter++; return(t); }); IEnumerable <int> query = Enumerable.Empty <int>(); switch (info.Operator) { case "AND": query = tag1Query.Intersect(tag2Query); if (tagsToExclude != null) { query = AddExclusionsToQuery(query, tagsToExclude, exclusionCounter); } break; case "AND-NOT": query = tag1Query.Except(tag2Query); if (tagsToExclude != null) { query = AddExclusionsToQuery(query, tagsToExclude, exclusionCounter); } break; case "OR": // TODO this has a small bug, we can get items out of order as we pull them thru in pairs // if t2 has several items that are larger than t1, t1 will still come out first!! // So algorithm needs to be: // 1) pull the LARGEST value (from t1 or t2) // 2) process this item // 3) repeat 1) again query = tag1Query.Zip(tag2Query, (t1, t2) => new[] { t1, t2 }) .SelectMany(item => item) .Distinct(); if (tagsToExclude != null) { query = AddExclusionsToQuery(query, tagsToExclude, exclusionCounter); } break; case "OR-NOT": //"i.e. .net+or+jquery-" // TODO this has a small bug, we can get items out of order as we pull them thru in pairs // if t2 has several items that are larger than t1, t1 will still come out first!! // So algorithm needs to be: // 1) pull the LARGEST value (from t1 or t2) // 2) process this item // 3) repeat 1) again query = tag1Query.Zip(queryInfo[TagServer.ALL_TAGS_KEY], (t1, t2) => new[] { t1, t2 }) .SelectMany(item => item) .Except(tag2Query) .Distinct(); if (tagsToExclude != null) { query = AddExclusionsToQuery(query, tagsToExclude, exclusionCounter); } break; // TODO Work out what a "NOT" query really means, at the moment it's the same as "AND-NOT"?! //case "NOT": // query = tag1Query.Except(tag2Query); // if (tagsToExclude != null) // query = AddExclusionsToQuery(query, tagsToExclude, exclusionCounter); // break; default: throw new InvalidOperationException(string.Format("Invalid operator specified: {0}", info.Operator ?? "<NULL>")); } var results = query.Skip(info.Skip) .Take(info.PageSize) .Select(i => questions[i]) .ToList(); timer.Stop(); Results.AddData(timer.Elapsed.TotalMilliseconds.ToString("#.##")); Logger.Log("REGULAR Boolean Query: \"{0}\" {1} \"{2}\", pageSize = {3:N0}, skip = {4:N0}, took {5} ({6:N2} ms) REGULAR", info.Tag, info.Operator, info.OtherTag, info.PageSize, info.Skip, timer.Elapsed, timer.Elapsed.TotalMilliseconds); Logger.Log("Got {0:} results in total, tag1 QueryCounter = {1:N0}, tag2 QueryCounter = {1:N0}", results.Count(), tagCounter, otherTagCounter); //PrintResults(results, string.Format("{0} {1} {2}", info.Tag, info.Operator, info.OtherTag), info.Type); return(new QueryResult { Questions = results, Counters = new Dictionary <string, int> { { "TagCounter", tagCounter }, { "OtherTagCounter", otherTagCounter }, { "ExclusionCounter", exclusionCounter.Counter } } }); }
internal QueryResult Query(QueryInfo info, EwahCompressedBitArray exclusionBitMap = null, bool printLoggingMessages = false) { var bitMap = GetTagByQueryBitMapLookup(info.Type); var questionLookup = GetTagByQueryLookup(info.Type)[TagServer.ALL_TAGS_KEY]; // Calculating the Cardinality can be (is?) expensive, we don't want to do it in Queries unless we really need to!? bool calculateCardinality = true; // false if (printLoggingMessages) { Logger.Log("Tag \"{0}\" is in {1:N0} Questions, Tag \"{2}\" is in {3:N0} Questions", info.Tag, allTags[info.Tag], info.OtherTag, allTags[info.OtherTag]); } //PrintResults(Enumerable.Range(0, questionLookup.Length), qu => questionLookup[qu], TagServer.ALL_TAGS_KEY, info.Type); //PrintResults(bitMap[info.Tag], qu => questionLookup[qu], info.Tag, info.Type); //PrintResults(bitMap[info.OtherTag], qu => questionLookup[qu], info.OtherTag, info.Type); var timer = Stopwatch.StartNew(); var tag1BitMap = bitMap[info.Tag]; var tag2BitMap = bitMap[info.OtherTag]; EwahCompressedBitArray bitMapResult = new EwahCompressedBitArray(); switch (info.Operator) { case "AND": bitMapResult = tag1BitMap.And(tag2BitMap); break; case "AND-NOT": bitMapResult = tag1BitMap.AndNot(tag2BitMap); break; case "OR": bitMapResult = tag1BitMap.Or(tag2BitMap); break; case "OR-NOT": //"i.e. .net+or+jquery-" bitMapResult = tag1BitMap.OrNot(tag2BitMap); break; // TODO Work out what a "NOT" query really means, the LINQ version was "result = tag1Query.Except(tag2Query)" (which is the same as AND-NOT?!) //case "NOT": // var bitMapResult = (EwahCompressedBitArray)tag2BitMap.Clone(); // bitMapResult.Not(); // break; default: throw new InvalidOperationException(string.Format("Invalid operator specified: {0}", info.Operator ?? "<NULL>")); } if (exclusionBitMap != null) { ulong cardinalityBeforeExclusions = 0; if (printLoggingMessages) { cardinalityBeforeExclusions = bitMapResult.GetCardinality(); } // The Exclusiong BitMap is Set (i.e. 1) in places where you CAN use the question, i.e. it's NOT excluded // That way we can efficiently apply the exclusions by ANDing this BitMap to the previous results var exclusionTimer = Stopwatch.StartNew(); bitMapResult = bitMapResult.AndNot(exclusionBitMap); exclusionTimer.Stop(); if (printLoggingMessages) { if (calculateCardinality) { Logger.Log("Took {0,5:N2} ms to apply exclusion BitMap (Cardinality={1:N0}), Results Cardinality: Before={2:N0}, After={3:N0}", exclusionTimer.Elapsed.TotalMilliseconds, exclusionBitMap.GetCardinality(), cardinalityBeforeExclusions, bitMapResult.GetCardinality()); } else { Logger.Log("Took {0,5:N2} ms to apply exclusion BitMap", exclusionTimer.Elapsed.TotalMilliseconds); } } } var resultCollectionTimer = Stopwatch.StartNew(); var result = bitMapResult.Skip(info.Skip) .Take(info.PageSize) .Select(i => questions[questionLookup[i]]) .ToList(); resultCollectionTimer.Stop(); if (printLoggingMessages) { Logger.Log("Took {0,5:N2} ms to collect the results", resultCollectionTimer.Elapsed.TotalMilliseconds); } timer.Stop(); Results.AddData(timer.Elapsed.TotalMilliseconds.ToString("#.##")); if (printLoggingMessages) { using (Utils.SetConsoleColour(ConsoleColor.DarkYellow)) { if (calculateCardinality) { Logger.Log("Took {0,5:N2} ms in TOTAL to calculate \"{1} {2} {3}\", Got {4} results, (Result Cardinality={5:N0})", timer.Elapsed.TotalMilliseconds, info.Tag, info.Operator, info.OtherTag, result.Count, bitMapResult.GetCardinality()); } else { Logger.Log("Took {0,5:N2} ms in TOTAL to calculate \"{1} {2} {3}\", Got {4} results", timer.Elapsed.TotalMilliseconds, info.Tag, info.Operator, info.OtherTag, result.Count); } } //PrintResults(bitMapResult, qu => questionLookup[qu], string.Format("{0} {1} {2}", info.Tag, info.Operator, info.OtherTag), info.Type); Logger.Log(); } return(new QueryResult { Questions = result, // TODO see if we can get meaningful numbers here, WITHOUT calling GetCardinality() (because it's expensive) //Counters = new Dictionary<string, int> //{ // { "TagCounter", tagCounter }, // { "OtherTagCounter", otherTagCounter }, // { "ExclusionCounter", exclusionCounter.Counter } //} }); }