protected override void GetToken()
{
base.GetToken();
// tokenise special tokens according to this language
// multi-character special tokens can also be added by checking scanner.curr for continuation
if (token.Type == TokenType.Special)
{
SpecialToken st = token as SpecialToken;
switch (st.Token)
{
case '=':
token.Type = tokenEquals;
break;
case ';':
token.Type = tokenSemicolon;
break;
case '+':
token.Type = tokenPlus;
break;
case '-':
token.Type = tokenMinus;
break;
case '/':
token.Type = tokenDivide;
break;
case '*':
token.Type = tokenMultiply;
break;
case '(':
token.Type = tokenOpenParentheses;
break;
case ')':
token.Type = tokenCloseParentheses;
break;
}
}
else if (token.Type == TokenType.Identifier)
{
// rule out identifiers that are actually reserved words. could be const values like "pi" or logic keywords like "if/then/else/while etc."
WordToken wt = token as WordToken;
switch (wt.Word)
{
case "pi":
token.Type = tokenPi;
break;
}
}
}
public static SpecialToken GetToken(Scanner scanner)
{
SpecialToken t = new SpecialToken();
t.sc = scanner.col;
t.sr = scanner.row;
t.Token = scanner.curr;
scanner.Next();
t.ec = scanner.col;
t.er = scanner.row;
return(t);
}
public static SpecialToken GetToken(Scanner scanner)
{
SpecialToken t = new SpecialToken();
t.sc = scanner.col;
t.sr = scanner.row;
// TODO: decide what special tokens will look like - this will be language specific so should be higher level IMO then the tokeniser can be unbiased
t.Token = scanner.curr;
scanner.Next();
t.ec = scanner.col;
t.er = scanner.row;
return(t);
}
// TODO: maintain a list of token types provided by parser so that scanner can remain unbiased
// TODO: maintain a list of special token types
public Token GetToken(bool skipWhitespace = true)
{
if (skipWhitespace)
{
SkipWhitespace();
}
if (currType == CharType.Whitespace)
{
return(new Token()
{
Type = TokenType.Whitespace
});
}
else if (currType == CharType.Alpha)
{
WordToken token = WordToken.GetToken(this);
// TODO: check for reserved words
return(token);
}
else if (currType == CharType.Numeric)
{
// TODO: need a scanner options class that lets me specify things like
// skipping whitespace, treating numbers etc.
// because here an IP address will be treated as a decimal number
// 192.0[.2.1]
return(NumberToken.GetToken(this));
}
else if (currType == CharType.Quote)
{
return(StringToken.GetToken(this));
}
else if (currType == CharType.EOF)
{
return(new EndOfFileToken());
}
else if (currType == CharType.Special)
{
SpecialToken token = SpecialToken.GetToken(this);
// TODO: check for special token types
return(token);
}
return(null);
}
protected override void GetToken(bool skipWhitespace = true)
{
base.GetToken(skipWhitespace);
if (token.Type == TokenType.Special)
{
SpecialToken st = token as SpecialToken;
if (st.Token == ';')
{
// we don't care about comments, keep scanning and return a real token
token.Type = tokenSemicolon;
do
{
scanner.Next();
}while (scanner.curr != '\n');
scanner.Next();
GetToken(skipWhitespace);
}
else if (st.Token == '$')
{
token.Type = tokenControl;
}
else if (st.Token == '(')
{
// TODO: I think don't ignore these, but i need to know when they will come
// so far SOA and WKS records, but if they could come anywhere then
// i need to cater for that
// WKS is particularly problematic because it isn't fixed length fields
//token.Type = tokenOpenParentheses;
// i think just ignore parentheses
GetToken(skipWhitespace);
}
else if (st.Token == ')')
{
// token.Type = tokenCloseParentheses;
GetToken(skipWhitespace);
}
else if (st.Token == '.')
{
token.Type = tokenDot;
}
else if (st.Token == ':')
{
token.Type = tokenColon;
}
else if (st.Token == '@')
{
token.Type = tokenAt;
}
}
else if (token.Type == TokenType.Identifier)
{
WordToken wt = token as WordToken;
if (wt.Word == "INCLUDE")
{
token.Type = tokenInclude;
}
else if (wt.Word == "ORIGIN")
{
token.Type = tokenOrigin;
}
else if (wt.Word == "TTL")
{
token.Type = tokenTTL;
}
else if (wt.Word == "IN")
{
token.Type = tokenInternet;
}
else if (wt.Word == "CS")
{
token.Type = tokenCSNET;
}
else if (wt.Word == "CH")
{
token.Type = tokenChaos;
}
else if (wt.Word == "HS")
{
token.Type = tokenHesiod;
}
else if (wt.Word == "AFXR")
{
token.Type = tokenTransfer;
}
else if (wt.Word == "MAILB")
{
token.Type = tokenMailbox;
}
else if (wt.Word == "MAILA")
{
token.Type = tokenMailAgent;
}
else if (wt.Word == "A")
{
token.Type = tokenHostAddress;
}
else if (wt.Word == "AAAA")
{
token.Type = tokenHostIPv6Address;
}
else if (wt.Word == "NS")
{
token.Type = tokenNameServer;
}
else if (wt.Word == "MD")
{
token.Type = tokenMailDestination;
}
else if (wt.Word == "MF")
{
token.Type = tokenMailForwarder;
}
else if (wt.Word == "CNAME")
{
token.Type = tokenCanonicalName;
}
else if (wt.Word == "SOA")
{
token.Type = tokenAuthority;
}
else if (wt.Word == "WKS")
{
token.Type = tokenWellKnownService;
}
else if (wt.Word == "PTR")
{
token.Type = tokenPointer;
}
else if (wt.Word == "HINFO")
{
token.Type = tokenHostInfo;
}
else if (wt.Word == "MINFO")
{
token.Type = tokenMailboxInfo;
}
else if (wt.Word == "MX")
{
token.Type = tokenMailExchange;
}
else if (wt.Word == "TXT")
{
token.Type = tokenText;
}
}
}
