private void AddCaseSensitiveToken(string text, bool needSeparators, object[] tags)
{
char startChar = text[0];
int startIndex = startChar;
if (nodes[startIndex] == null)
nodes[startIndex] = new TokenTreeNode();
nodes[startIndex].AddToken(text, true, needSeparators, tags);
}
private void AddPatternWithCaseSensitivePrefix(string prefix, IPatternMatcher matcher, bool needSeparators,
object[] tags)
{
char startChar = prefix[0];
int startIndex = startChar;
if (nodes[startIndex] == null)
nodes[startIndex] = new TokenTreeNode();
nodes[startIndex].AddPattern(prefix, true, needSeparators, matcher, tags);
}
private void AddPatternWithCaseInsensitivePrefix(string prefix, IPatternMatcher matcher, bool needSeparators,
object[] tags)
{
//make a lowercase string and add it as a token
prefix = prefix.ToLower();
char startChar = prefix[0];
int startIndex = startChar;
if (nodes[startIndex] == null)
nodes[startIndex] = new TokenTreeNode();
nodes[startIndex].AddPattern(prefix, false, needSeparators, matcher, tags);
//make a lowercase string with a uppercase start char and add it as a token
prefix = char.ToUpper(startChar) + prefix.Substring(1);
startChar = prefix[0];
startIndex = startChar;
if (nodes[startIndex] == null)
nodes[startIndex] = new TokenTreeNode();
nodes[startIndex].AddPattern(prefix, false, needSeparators, matcher, tags);
}
private void AddCaseInsensitiveToken(string text, bool needSeparators, object[] tags)
{
//make a lowercase string and add it as a token
text = text.ToLower();
char startChar = text[0];
int startIndex = startChar;
if (nodes[startIndex] == null)
nodes[startIndex] = new TokenTreeNode();
nodes[startIndex].AddToken(text, false, needSeparators, tags);
//make a lowercase string with a uppercase start char and add it as a token
text = char.ToUpper(startChar) + text.Substring(1);
startChar = text[0];
startIndex = startChar;
if (nodes[startIndex] == null)
nodes[startIndex] = new TokenTreeNode();
nodes[startIndex].AddToken(text, false, needSeparators, tags);
}
public TokenTreeNode AddTokenInternal(string token, bool caseSensitive)
{
Char = token[0];
if (!caseSensitive)
ContainsCaseInsensitiveData = true;
if (token.Length == 1)
return this;
string leftovers = token.Substring(1);
char childChar = leftovers[0];
int childIndex = childChar & 0xff;
//make a lookupindex (dont mind if unicode chars end up as siblings as ascii)
TokenTreeNode node = ChildNodes[childIndex];
TokenTreeNode res;
if (node == null)
{
var child = new TokenTreeNode();
ChildNodes[childIndex] = child;
res = child.AddTokenInternal(leftovers, caseSensitive);
MakeRepeatingWS(child);
}
else
{
node = GetMatchingNode(childChar, node);
res = node.AddTokenInternal(leftovers, caseSensitive);
}
return res;
}
private static void MakeRepeatingWS(TokenTreeNode child)
{
if (child.Char == ' ')
{
// if the node contains " " (whitespace)
// then add the node as a childnode of itself.
// thus allowing it to parse things like
// "end sub" even if the pattern is "end sub" // do not localize
child.ChildNodes[' '] = child;
}
}
private static TokenTreeNode GetMatchingNode(char childChar, TokenTreeNode node)
{
//find a bucket with the same childChar as we need
while (node.NextSibling != null && node.Char != childChar)
{
node = node.NextSibling;
}
if (node.Char != childChar)
{
var child = new TokenTreeNode();
node.NextSibling = child;
return child;
}
return node;
}