protected virtual GeneroToken GetCommentToken()
{
GeneroToken retToken = null;
// Test for comment
if (_source[_currentPosition] == '#' ||
(_source[_currentPosition] == '-' &&
(_source.Length > _currentPosition + 1 && _source[_currentPosition + 1] == '-')))
{
// go all the way to the end of the line
while (_currentPosition < _source.Length &&
(_source[_currentPosition] != '\r' &&
_source[_currentPosition] != '\n'))
{
IncrementCurrentPosition();
}
string temp = _source.Substring(_tokenStart, _currentPosition - _tokenStart);
retToken = new GeneroToken(temp,
_tokenStart,
_currentPosition,
TokenCategory.Comment,
_currentLineNumber,
_currentColumn - temp.Length);
//_currentPosition++;
}
return(retToken);
}
protected virtual GeneroToken EnsureNotEndOfFile()
{
if (IsEof)
{
var token = new GeneroToken("", _currentPosition, _currentPosition, TokenCategory.EndOfStream, _currentLineNumber, _currentColumn);
return(token);
}
return(null);
}
public GeneroToken NextToken()
{
GeneroToken retToken = null;
retToken = EnsureNotEndOfFile();
if (retToken != null)
{
return(retToken);
}
if (CanGetTokens())
{
AdvancePastWhitespace();
retToken = EnsureNotEndOfFile();
if (retToken != null)
{
return(retToken);
}
_tokenStart = _currentPosition;
if ((retToken = GetCommentToken()) != null)
{
return(retToken);
}
if ((retToken = GetNumberToken()) != null)
{
return(retToken);
}
if ((retToken = GetStringToken()) != null)
{
return(retToken);
}
if ((retToken = GetSymbolToken()) != null)
{
return(retToken);
}
if ((retToken = GetKeywordOrIdentifierToken()) != null)
{
return(retToken);
}
// if we didn't find anything, we'll want to advance
if (retToken == null)
{
IncrementCurrentPosition();
}
return(new GeneroToken("", 0, 0, TokenCategory.None, 0, 0));
}
return(new GeneroToken("", 0, 0, TokenCategory.None, 0, 0));;
}
protected virtual GeneroToken GetStringToken()
{
GeneroToken retToken = null;
// Test for string
bool doubleQuote = _source[_currentPosition] == '"';
bool singleQuote = _source[_currentPosition] == '\'';
if (doubleQuote || singleQuote)
{
IncrementCurrentPosition();
while (_currentPosition < _source.Length &&
(
(doubleQuote && _source[_currentPosition] != '"') ||
(singleQuote && _source[_currentPosition] != '\'')))
{
if (_source[_currentPosition] == '\\' &&
_currentPosition + 1 < _source.Length)
{
// skip the escaped character
IncrementCurrentPosition();
}
// TODO: at some point it would be nice to support multi-line strings...but not yet
if (_source[_currentPosition] == '\r' || _source[_currentPosition] == '\n')
{
break;
}
IncrementCurrentPosition();
}
if (_currentPosition < _source.Length)
{
IncrementCurrentPosition();
}
// if there's nothing beyond the string, we need to decrease the current position so we don't throw an exception
while (_currentPosition > _source.Length)
{
_currentPosition--;
}
string temp = "";
if (_currentPosition >= _tokenStart)
{
temp = _source.Substring(_tokenStart, _currentPosition - _tokenStart);
}
retToken = new GeneroToken(temp,
_tokenStart,
_currentPosition,
TokenCategory.StringLiteral,
_currentLineNumber,
_currentColumn - temp.Length);
}
return(retToken);
}
protected virtual GeneroToken GetKeywordOrIdentifierToken()
{
GeneroToken retToken = null;
if (_source[_currentPosition] == '_' ||
char.IsLetter(_source[_currentPosition]))
{
IncrementCurrentPosition();
while (_currentPosition < _source.Length &&
(_source[_currentPosition] == '_' ||
char.IsLetter(_source[_currentPosition]) ||
char.IsDigit(_source[_currentPosition])))
{
IncrementCurrentPosition();
}
// look for it in the keyword map
string normalCase = _source.Substring(_tokenStart, _currentPosition - _tokenStart);
string lowercase = normalCase.ToLower();
if (Tokens.GetToken(lowercase) != null)
{
retToken = new GeneroToken(normalCase,
_tokenStart,
_currentPosition,
TokenCategory.Keyword,
_currentLineNumber,
_currentColumn - normalCase.Length);
//_currentPosition++;
return(retToken);
}
retToken = new GeneroToken(normalCase,
_tokenStart,
_currentPosition,
TokenCategory.Identifier,
_currentLineNumber,
_currentColumn - normalCase.Length);
//_currentPosition++;
return(retToken);
}
return(retToken);
}
public GeneroToken Lookahead(int numTokens)
{
int backupPos = _currentPosition;
int backupLine = _currentLineNumber;
int backupCol = _currentColumn;
GeneroToken token = null;
for (int i = 0; i < numTokens; i++)
{
token = NextToken();
if (token.TokenType == TokenCategory.None)
{
break;
}
}
_currentPosition = backupPos;
_currentLineNumber = backupLine;
_currentColumn = backupCol;
return(token);
}
protected virtual GeneroToken GetSymbolToken()
{
GeneroToken retToken = null;
// Look for symbols
int backupPos = _currentPosition; // need to save the current position, in case of false initial matches
if (IsSymbol(_source[_currentPosition]))
{
IncrementCurrentPosition();
// need to handle tokens that have 2 (or more) symbols
//if (_currentPosition < _source.Length &&
// IsSymbol(_source[_currentPosition]))
//{
// Tokens.GetToken( _lexerDefinition.SymbolMap.ContainsKey())
//{
// var oneAhead = _source.Substring(_tokenStart, _currentPosition + 1 - _tokenStart);
// var tok = Tokens.GetToken(oneAhead);
// IncrementCurrentPosition();
//}
//// look in the symbol map
//if (_lexerDefinition.SymbolMap.ContainsKey(_source.Substring(_tokenStart, _currentPosition - _tokenStart)))
//{
string temp = _source.Substring(_tokenStart, _currentPosition - _tokenStart);
retToken = new GeneroToken(temp,
_tokenStart,
_currentPosition,
TokenCategory.Operator,
_currentLineNumber,
_currentColumn - temp.Length);
//_currentPosition++;
return(retToken);
//}
}
_currentPosition = backupPos;
return(retToken);
}
protected virtual GeneroToken GetNumberToken()
{
GeneroToken retToken = null;
// Test for number
if (IsNumberCharacter())
{
IncrementCurrentPosition();
while (_currentPosition < _source.Length &&
IsNumberCharacter())
{
IncrementCurrentPosition();
}
string temp = _source.Substring(_tokenStart, _currentPosition - _tokenStart);
retToken = new GeneroToken(temp,
_tokenStart,
_currentPosition,
TokenCategory.NumericLiteral,
_currentLineNumber,
_currentColumn - temp.Length);
//_currentPosition++;
}
return(retToken);
}