public Token[] Tokenize()
{
if (Text == null) {
throw new ArgumentNullException("Text");
}
MakeImmutable();
var tokens = new List<Token>();
int index = 0;
while (index < Text.Length) {
MatchResult match = tree.Match(Text, index);
if (match.Found) {
string dummyText = Text.Substring(index, match.Index - index);
var dummyToken = new Token(dummyText, null);
tokens.Add(dummyToken);
var realToken = new Token(match.GetText(), match.Tags);
index = match.Index + match.Length;
tokens.Add(realToken);
} else {
string dummyText = Text.Substring(index);
var dummyToken = new Token(dummyText, null);
tokens.Add(dummyToken);
index = Text.Length;
}
}
return tokens.ToArray();
}
public Token[] Tokenize()
{
Text.Require("Text")
.NotNullOrEmpty();
MakeImmutable();
var tokens = new List<Token>();
int index = 0;
while (index < Text.Length)
{
MatchResult match = tree.Match(Text, index);
if (match.Found)
{
string dummyText = Text.Substring(index, match.Index - index);
var dummyToken = new Token(dummyText, null);
tokens.Add(dummyToken);
var realToken = new Token(match.GetText(), match.Tags);
index = match.Index + match.Length;
tokens.Add(realToken);
}
else
{
string dummyText = Text.Substring(index);
var dummyToken = new Token(dummyText, null);
tokens.Add(dummyToken);
index = Text.Length;
}
}
return tokens.ToArray();
}
