private static void TestWildcards(TagServer tagServer, NGrams nGrams, params string[] tagsToExpandInput)
{
var tagsToExpand = tagsToExpandInput.ToList();
if (tagsToExpand.Count == 1 && tagsToExpand[0] == "*")
{
// special case!!
using (Utils.SetConsoleColour(ConsoleColor.Green))
Logger.Log("\nTestWildcards: special case, using ALL Tags", String.Join(", ", tagsToExpand));
var bitMapIndex = tagServer.CreateBitMapIndexForExcludedTags(new CLR.HashSet <string>(tagServer.AllTags.Keys), QueryType.AnswerCount, printLoggingMessages: true);
return;
}
using (Utils.SetConsoleColour(ConsoleColor.Green))
Logger.Log("\nTestWildcards: {0}\n", String.Join(", ", tagsToExpand.Where(t => t.Contains('*'))));
var timer = Stopwatch.StartNew();
var expandedTagsNGrams = WildcardProcessor.ExpandTagsNGrams(tagServer.AllTags, tagsToExpand, nGrams, printLoggingMessages: true);
timer.Stop();
using (Utils.SetConsoleColour(ConsoleColor.DarkYellow))
{
Logger.LogStartupMessage("Took {0,6:N2} ms ({1}) to expanded Wildcards to {2,2:N0} tags (using N-Grams, with N={3})",
timer.Elapsed.TotalMilliseconds, timer.Elapsed, expandedTagsNGrams.Count, WildcardProcessor.N);
}
var wildcards = tagsToExpand.Where(t => t.Contains('*')).ToList();
Logger.LogStartupMessage("There are {0:N0} wildcards in the list and {1:N0} regular tags (i.e. with no '*' in them)",
wildcards.Count, tagsToExpand.Count(w => w.Contains('*') == false));
if (wildcards.Count > 50)
{
Logger.LogStartupMessage("Wildcards: TOO MANY TO PRINT (there are {0:N0} wildcards)", wildcards.Count);
}
else
{
Logger.LogStartupMessage("Wildcards: [{0}]", String.Join(", ", tagsToExpand.Where(w => w.Contains('*'))));
}
var expansions = tagsToExpand.Where(w => w.Contains('*'))
.Select(w => String.Format("{0} -> {1}", w, String.Join(", ", WildcardProcessor.CreateSearches(w))))
.ToList();
if (expansions.Count > 50)
{
Logger.LogStartupMessage("Expansions: TOO MANY TO PRINT (there are {0:N0} expansions)", expansions.Count);
}
else
{
Logger.LogStartupMessage("Expansions:\n {0}", String.Join("\n ", expansions));
}
if (expandedTagsNGrams.Count > 50)
{
Logger.LogStartupMessage("Results: TOO MANY TO PRINT (there are {0:N0} results)", expandedTagsNGrams.Count);
}
else
{
Logger.LogStartupMessage("Results: [{0}]", String.Join(", ", expandedTagsNGrams));
}
var expandTagsContainsTimer = Stopwatch.StartNew();
var expandTagsContains = WildcardProcessor.ExpandTagsContainsStartsWithEndsWith(tagServer.AllTags, tagsToExpand);
expandTagsContainsTimer.Stop();
Logger.LogStartupMessage("\nIn Contains but not in NGrams: " + string.Join(", ", expandTagsContains.Except(expandedTagsNGrams)));
Logger.LogStartupMessage("\nIn NGrams but not in Contains: " + string.Join(", ", expandedTagsNGrams.Except(expandTagsContains)));
Logger.LogStartupMessage();
var bitMapIndexAnswerCount = tagServer.CreateBitMapIndexForExcludedTags(expandedTagsNGrams, QueryType.AnswerCount, printLoggingMessages: true);
//var bitMapIndexCreationDate = tagServer.CreateBitMapIndexForExcludedTags(expandedTagsNGrams, QueryType.CreationDate, printLoggingMessages: true);
//var bitMapIndexLastActivityDate = tagServer.CreateBitMapIndexForExcludedTags(expandedTagsNGrams, QueryType.LastActivityDate, printLoggingMessages: true);
//var bitMapIndexScore = tagServer.CreateBitMapIndexForExcludedTags(expandedTagsNGrams, QueryType.Score, printLoggingMessages: true);
//var bitMapIndexViewCount = tagServer.CreateBitMapIndexForExcludedTags(expandedTagsNGrams, QueryType.ViewCount, printLoggingMessages: true);
}
private static HashSet ProcessTagsForFastLookup(TagLookup allTags, Trie <int> trie, NGrams nGrams, List <string> tagsToExpand)
{
var expandTagsContainsTimer = Stopwatch.StartNew();
var expandTagsContains = WildcardProcessor.ExpandTagsContainsStartsWithEndsWith(allTags, tagsToExpand);
expandTagsContainsTimer.Stop();
//var expandTagsVBTimer = Stopwatch.StartNew();
//var expandedTagsVB = WildcardProcessor.ExpandTagsVisualBasic(allTags, tagsToExpand);
//expandTagsVBTimer.Stop();
var expandTagsRegexTimer = Stopwatch.StartNew();
var expandedTagsRegex = WildcardProcessor.ExpandTagsRegex(allTags, tagsToExpand);
expandTagsRegexTimer.Stop();
var expandTagsTrieTimer = Stopwatch.StartNew();
var expandedTagsTrie = WildcardProcessor.ExpandTagsTrie(allTags, tagsToExpand, trie);
expandTagsTrieTimer.Stop();
var expandedTagsNGramsTimer = Stopwatch.StartNew();
var expandedTagsNGrams = WildcardProcessor.ExpandTagsNGrams(allTags, tagsToExpand, nGrams, printLoggingMessages: true);
expandTagsRegexTimer.Stop();
Logger.LogStartupMessage("\nThere are {0:N0} tags in total", allTags.Count);
Logger.LogStartupMessage("There are {0:N0} tags/wildcards (raw) BEFORE expansion", tagsToExpand.Count);
Logger.LogStartupMessage("\nExpanded to {0,4:N0} tags (Contains), took {1,8:N2} ms ({2})",
expandTagsContains.Count, expandTagsContainsTimer.Elapsed.TotalMilliseconds, expandTagsContainsTimer.Elapsed);
//Logger.LogStartupMessage("Expanded to {0,4:N0} tags (VB), took {1,8:N2} ms ({2})",
// expandedTagsVB.Count, expandTagsVBTimer.Elapsed.TotalMilliseconds, expandTagsVBTimer.Elapsed);
Logger.LogStartupMessage("Expanded to {0,4:N0} tags (Regex), took {1,8:N2} ms ({2})",
expandedTagsRegex.Count, expandTagsRegexTimer.Elapsed.TotalMilliseconds, expandTagsRegexTimer.Elapsed);
Logger.LogStartupMessage("Expanded to {0,4:N0} tags (Trie), took {1,8:N2} ms ({2})",
expandedTagsTrie.Count, expandTagsTrieTimer.Elapsed.TotalMilliseconds, expandTagsTrieTimer.Elapsed);
Logger.LogStartupMessage("Expanded to {0,4:N0} tags (N-Grams), took {1,8:N2} ms ({2})",
expandedTagsNGrams.Count, expandedTagsNGramsTimer.Elapsed.TotalMilliseconds, expandedTagsNGramsTimer.Elapsed);
Logger.LogStartupMessage("\nIn Contains but not in Regex: " + string.Join(", ", expandTagsContains.Except(expandedTagsRegex)));
Logger.LogStartupMessage("\nIn Regex but not in Contains: " + string.Join(", ", expandedTagsRegex.Except(expandTagsContains)));
//Logger.LogStartupMessage("\nIn Contains but not in VB: " + string.Join(", ", expandTagsContains.Except(expandedTagsVB)));
//Logger.LogStartupMessage("\nIn VB but not in Contains: " + string.Join(", ", expandedTagsVB.Except(expandTagsContains)));
Logger.LogStartupMessage("\nIn Contains but not in Trie: " + string.Join(", ", expandTagsContains.Except(expandedTagsTrie)));
Logger.LogStartupMessage("\nIn Trie but not in Contains: " + string.Join(", ", expandedTagsTrie.Except(expandTagsContains)));
Logger.LogStartupMessage("\nIn Contains but not in NGrams: " + string.Join(", ", expandTagsContains.Except(expandedTagsNGrams)));
Logger.LogStartupMessage("\nIn NGrams but not in Contains: " + string.Join(", ", expandedTagsNGrams.Except(expandTagsContains)));
Logger.LogStartupMessage();
var expandedTags = expandedTagsNGrams;
//Logger.LogStartupMessage(string.Join(", ", expandedTags));
// This is an error, we shouldn't have extra tags that aren't in the "allTags" list!!
var extra = expandedTags.Except(allTags.Keys).ToList();
if (extra.Count > 0)
{
Logger.LogStartupMessage("\nExtra Tags: " + string.Join(", ", extra) + "\n");
}
return(expandedTags);
}
