/// <summary>
/// Gets the next token from the input.
/// </summary>
/// <param name="token">Current token.</param>
/// <returns>The next token from the input.</returns>
internal Token Next(Token token)
{
// do we already know the next token?
if (token != null && token.Link != null)
return token.Link;
// start scanning at the position of the last token
var pos = token != null ? token.Position.Offset + token.Position.Length : 0;
// scan for the next token
Token next = null;
while (true) {
// skip leading whitespace
var ch = GetChar(pos);
while (char.IsWhiteSpace(ch))
ch = GetChar(++pos);
// are we at the end of the input?
if (pos >= _input.Length) {
next = MakeToken(Tokens.End);
next.Position = new Position(pos, 0);
if (token != null)
token.Link = next;
break;
}
// deal with the various input characters
var start = pos;
switch (ch) {
// character and string literals
case '\'':
next = ScanCharacter(ref pos);
break;
case '"':
next = ScanString(ref pos);
break;
// punctuation
case '.':
pos++;
next = MakeToken(Tokens.Dot);
break;
case ',':
pos++;
next = MakeToken(Tokens.Comma);
break;
case ':':
pos++;
next = MakeToken(Tokens.Colon);
break;
case ';':
pos++;
next = MakeToken(Tokens.Semi);
break;
case '(':
pos++;
next = MakeToken(Tokens.LeftParen);
break;
case ')':
pos++;
next = MakeToken(Tokens.RightParen);
break;
case '{':
pos++;
next = MakeToken(Tokens.LeftBrace);
break;
case '}':
pos++;
next = MakeToken(Tokens.RightBrace);
break;
case '[':
pos++;
next = MakeToken(Tokens.LeftBracket);
break;
case ']':
pos++;
next = MakeToken(Tokens.RightBracket);
break;
// operators
case '=':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.DoubleEquals);
} else if (ch == '>') {
pos++;
next = MakeToken(Tokens.EqualsRightChevron);
} else {
next = MakeToken(Tokens.Equals);
}
break;
case '+':
ch = GetChar(++pos);
if (ch == '+') {
pos++;
next = MakeToken(Tokens.DoublePlus);
} else if (ch == '=') {
pos++;
next = MakeToken(Tokens.PlusEquals);
} else {
next = MakeToken(Tokens.Plus);
}
break;
case '-':
ch = GetChar(++pos);
if (ch == '-') {
pos++;
next = MakeToken(Tokens.DoubleMinus);
} else if (ch == '=') {
pos++;
next = MakeToken(Tokens.MinusEquals);
} else {
next = MakeToken(Tokens.Minus);
}
break;
case '*':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.AsteriskEquals);
} else {
next = MakeToken(Tokens.Asterisk);
}
break;
case '/':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.SlashEquals);
} else {
next = MakeToken(Tokens.Slash);
}
break;
case '%':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.PercentEquals);
} else {
next = MakeToken(Tokens.Percent);
}
break;
case '!':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.ExclamationEquals);
} else {
next = MakeToken(Tokens.Exclamation);
}
break;
case '?':
ch = GetChar(++pos);
if (ch == '?') {
pos++;
next = MakeToken(Tokens.DoubleQuestion);
} else {
next = MakeToken(Tokens.Question);
}
break;
case '~':
next = MakeToken(Tokens.Tilde);
break;
case '&':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.AmpersandEquals);
} else if (ch == '&') {
pos++;
next = MakeToken(Tokens.DoubleAmpersand);
} else {
next = MakeToken(Tokens.Ampersand);
}
break;
case '|':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.BarEquals);
} else if (ch == '|') {
pos++;
next = MakeToken(Tokens.DoubleBar);
} else {
next = MakeToken(Tokens.Bar);
}
break;
case '^':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.CaretEquals);
} else {
next = MakeToken(Tokens.Caret);
}
break;
case '<':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.LeftChevronEquals);
} else if (ch == '<') {
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.DoubleLeftChevronEquals);
} else {
next = MakeToken(Tokens.DoubleLeftChevron);
}
} else {
next = MakeToken(Tokens.LeftChevron);
}
break;
case '>':
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.RightChevronEquals);
} else if (ch == '>') {
ch = GetChar(++pos);
if (ch == '=') {
pos++;
next = MakeToken(Tokens.DoubleRightChevronEquals);
} else {
next = MakeToken(Tokens.DoubleRightChevron);
}
} else {
next = MakeToken(Tokens.RightChevron);
}
break;
default:
if (IsIdentifierStart(ch) || ch == '\\')
next = ScanIdentifier(ref pos);
else if (IsDecimalDigit(ch))
next = ScanNumber(ref pos);
else {
Error(new Position(pos, 1), Messages.UnexpectedInput(ch));
pos++;
continue;
}
break;
}
// set position and attach to preceding token
next.Position = new Position(start, pos - start);
if (token != null)
token.Link = next;
break;
}
return next;
}
/// <summary>
/// Place all tokens from the given token and up to and including the end on the free-list, making them ready for re-use.
/// </summary>
/// <param name="token">First token to free</param>
/// <param name="end">Last token to free</param>
private void Free(Token token, Token end)
{
while (token != end) {
var next = token.Link;
Free(token);
token = next;
}
}
/// <summary>
/// Make a new token.
/// </summary>
/// <param name="kind">Kind of token to make.</param>
/// <param name="value">Value to associate with the token.</param>
private Token MakeToken(Tokens kind, object value = null)
{
// re-use a token off the free-list, or allocate a new one
Token token;
if (_freeList != null) {
token = _freeList;
_freeList = _freeList.Link;
} else {
token = new Token();
}
// initialise it
token.Kind = kind;
token.Value = value;
token.Link = null;
return token;
}
/// <summary>
/// Place the given token on the free-list, making it ready for re-use.
/// </summary>
/// <param name="token">Token to free.</param>
private void Free(Token token)
{
token.Link = _freeList;
_freeList = token;
}
