public TToken Get()
{
TToken token;
skipWhiteSpace();
//--Determine the token class, based on the current character.
switch (CharCodeMapManager.Instance.GetCharacterMap()[buffer.CurrentChar])
{
case CharCode.Letter: token = new WordToken(); break;
case CharCode.Digit: token = new NumberToken(); break;
case CharCode.Quote: token = new StringToken(); break;
case CharCode.Special: token = new SpecialToken(); break;
case CharCode.EndOfFile: token = new EOFToken(); break;
default: if (buffer.CurrentPosition == 0 && buffer.CurrentChar == 0) token = new EOFToken(); else token = new ErrorToken(); break;
}
//--Extract a token of that class, and return a pointer to it.
token.Get(buffer);
return token;
}
public TToken Get()
{
TToken pToken = null;
TTokenCode token;
int code;
//--Loop to process any line markers
//--and extract the next token code.
do
{
//--First read the token code.
code = m_formatter.ReadInt32();
token = (TTokenCode)code;
if (token == TTokenCode.LineMarker)
{
int number = m_formatter.ReadInt32();
m_currentLineNumber = number;
}
} while (token == TTokenCode.LineMarker);
//--Determine the token class, based on the token code.
switch (token)
{
case TTokenCode.Number: pToken = new NumberToken(); break;
case TTokenCode.String: pToken = new StringToken(); break;
case TTokenCode.Identifier:
pToken = new WordToken();
pToken.Code = TTokenCode.Identifier;
break;
default:
if (code < (int)TTokenCode.And)
{
pToken = new SpecialToken();
pToken.Code = token;
}
else
{
pToken = new WordToken(); // reserved word
pToken.Code = token;
}
break;
}
// now get node
//--Extract the symbol table node and set the token string.
switch (token)
{
case TTokenCode.Identifier:
case TTokenCode.Number:
case TTokenCode.String:
pNode = GetNode();
pToken.TokenString = pNode.String;
break;
default:
pNode = null;
pToken.TokenString = symbolStrings[code];
break;
}
return pToken;
}
