public static Queue <Token> TokenizeString(string buffer, string fileName = "")
{
var bufferLines = buffer.Split(new[] { Environment.NewLine }, StringSplitOptions.None);
MatchCollection matches = regexPattern.Matches(buffer);
Queue <Token> tokenList = new Queue <Token>();
foreach (Match match in matches)
{
int i = 0;
foreach (Group group in match.Groups)
{
string matchValue = group.Value;
bool success = group.Success;
if (success && i > 1)
{
string groupName = regexPattern.GroupNameFromNumber(i);
if (groupName == "comment")
{
continue; // don't create tokens for comments
}
int startLineNumber, startColumnNumber, endLineNumber, endColumnNumber;
CalculateTextOffset(bufferLines, match.Index, out startLineNumber, out startColumnNumber);
CalculateTextOffset(bufferLines, match.Index + match.Length, out endLineNumber, out endColumnNumber);
var tokenSource = new TokenSourceInfo(fileName, startLineNumber, startColumnNumber, endLineNumber, endColumnNumber);
var token = new Token(groupName, matchValue, tokenSource);
tokenList.Enqueue(token);
}
i++;
}
}
return(tokenList);
}
public static Queue<Token> TokenizeString( string buffer, string fileName = "" )
{
var bufferLines = buffer.Split( new[] { Environment.NewLine }, StringSplitOptions.None );
MatchCollection matches = regexPattern.Matches( buffer );
Queue<Token> tokenList = new Queue<Token>();
foreach ( Match match in matches )
{
int i = 0;
foreach ( Group group in match.Groups )
{
string matchValue = group.Value;
bool success = group.Success;
if ( success && i > 1 )
{
string groupName = regexPattern.GroupNameFromNumber( i );
if ( groupName == "comment" )
continue; // don't create tokens for comments
int startLineNumber, startColumnNumber, endLineNumber, endColumnNumber;
CalculateTextOffset( bufferLines, match.Index, out startLineNumber, out startColumnNumber);
CalculateTextOffset( bufferLines, match.Index + match.Length, out endLineNumber, out endColumnNumber);
var tokenSource = new TokenSourceInfo( fileName, startLineNumber, startColumnNumber, endLineNumber, endColumnNumber );
var token = new Token( groupName, matchValue, tokenSource );
tokenList.Enqueue( token );
}
i++;
}
}
return tokenList;
}
public Token( string name, string value, TokenSourceInfo source )
: this( name, value )
{
Source = source;
}
public Token(string name, string value, TokenSourceInfo source)
: this(name, value)
{
Source = source;
}