private void Tokenize(string nlChars, string line)
{
var lineBuilder = new LineBuilder(++_lineNo, _tokens);
int leading = LeadingSpacesOf(line);
lineBuilder.AddToken(nlChars, line.Substring(0, leading), NL);
line = line.Substring(leading);
if (line.Length != 0)
{
lineBuilder.AddToken("", line, NORMAL);
}
}
public static TokenChecker Of(string firstLine)
{
var tb = new TokenChecker();
var lineBuilder = new LineBuilder(0, tb._tokens);
int leading = LeadingSpacesOf(firstLine);
lineBuilder.Pos = leading;
firstLine = firstLine.Substring(leading);
if (firstLine.Length != 0)
{
lineBuilder.AddToken("", firstLine, NORMAL);
}
return(tb);
}