private Token ExtractToken()
{
SkipWhitespace();
char current = source.GetCurrentChar();
Token token;
if (current == Source.END_OF_FILE)
{
TokenBuilder builder = new TokenBuilder();
token = builder.CreateTokenWithType(TokenType.END_OF_FILE)
.AtLine(source.LineNumber)
.AtPosition(source.Position)
.Build();
} else if (Char.IsLetter(current))
{
token = ExtractWordToken();
} else if (Char.IsDigit(current))
{
token = ExtractNumberToken();
} else if (current == '\'')
{
token = ExtractStringToken();
} else if (TokenTypeInfo.IsSpecialSymbol(current.ToString()))
{
token = ExtractSpecialToken();
} else
{
TokenBuilder builder = new TokenBuilder();
token = builder.CreateTokenWithType(TokenType.ERROR)
.WithLexeme(current.ToString())
.WithValue(ErrorCode.INVALID_CHARACTER)
.AtLine(source.LineNumber)
.AtPosition(source.Position)
.Build();
source.GetNextChar();
}
return token;
}
private Token ExtractSpecialToken()
{
char current = source.GetCurrentChar();
StringBuilder lexeme = new StringBuilder();
lexeme.Append(current);
int pos = source.Position;
int line= source.LineNumber;
string value = null;
TokenType type = TokenType.INVALID;
switch (current) {
case '+': case '-': case '*': case '/': case ',':
case ';': case '\'': case '=': case '(': case ')':
case '[': case ']': case '{': case '}': case '^':
source.GetNextChar();
break;
case ':':
current = source.GetNextChar(); // consume ':'
if (current == '=')
{
lexeme.Append(current);
source.GetNextChar(); // consume '='
}
break;
case '<':
current = source.GetNextChar();
if (current == '=')
{
lexeme.Append(current);
source.GetNextChar();
} else if (current == '>')
{
lexeme.Append(current);
source.GetNextChar();
}
break;
case '>':
current = source.GetNextChar();
if (current == '=')
{
lexeme.Append(current);
source.GetNextChar();
}
break;
case '.':
current = source.GetNextChar();
if (current == '.')
{
lexeme.Append(current);
source.GetNextChar();
}
break;
default:
source.GetNextChar();
type = TokenType.ERROR;
value = ErrorCode.INVALID_CHARACTER.Message;
break;
}
if (type == TokenType.INVALID)
{
type = TokenTypeInfo.SpecialTypeFromString(lexeme.ToString());
}
TokenBuilder builder = new TokenBuilder();
return builder.CreateTokenWithType(type).WithLexeme(lexeme.ToString())
.WithValue(value)
.AtPosition(pos)
.AtLine(line).Build();
}
private Token ExtractStringToken()
{
int line = source.LineNumber;
int pos = source.Position;
StringBuilder lexeme = new StringBuilder();
StringBuilder value = new StringBuilder();
char current = source.GetNextChar(); // consume initial quote
lexeme.Append('\'');
// deal with string characters
do
{
// replace any whitespace character with a blank
if (Char.IsWhiteSpace(current))
{
current = ' ';
}
if (current != '\'' && current != Source.END_OF_FILE)
{
lexeme.Append(current);
value.Append(current);
current = source.GetNextChar(); // again consume
}
// quote ? each pair of adjacent quotes represent a single quote.
if (current == '\'')
{
while (current == '\'' && source.PeekNextChar() == '\'')
{
lexeme.Append("''");
value.Append('\'');
current = source.GetNextChar();
current = source.GetNextChar(); // consume both quotes
}
}
} while(current != '\'' && current != Source.END_OF_FILE);
TokenBuilder builder = new TokenBuilder();
if (current == '\'')
{
source.GetNextChar(); // consume final quote
lexeme.Append('\'');
builder.CreateTokenWithType(TokenType.STRING).WithValue(value.ToString()).WithLexeme(lexeme.ToString());
} else
{
builder.CreateTokenWithType(TokenType.ERROR).WithValue(ErrorCode.UNEXPECTED_EOF);
}
return builder.AtPosition(pos).AtLine(line).Build();
}
private Token ExtractNumberToken()
{
int line = source.LineNumber;
int pos = source.Position;
var uval = UnsignedDigits();
TokenType err = uval.Item1;
string lexeme = uval.Item2;
object value = uval.Item3;
string whole_digits = uval.Item4;
if (err == TokenType.ERROR)
{
TokenBuilder builder = new TokenBuilder();
return builder.CreateTokenWithType(err)
.WithLexeme(lexeme)
.AtPosition(pos)
.AtLine(line)
.WithValue(value)
.Build();
}
// assume int
TokenType type = TokenType.INTEGER;
// Is there a . ?
// It could be a decimal point or the start of a .. token
char current = source.GetCurrentChar();
bool saw_dot_dot = false;
string frac_digits = "";
if (current == '.')
{
if (source.PeekNextChar() == '.')
{
saw_dot_dot = true; // it's a ".." token, so don't consume it
} else
{
type = TokenType.REAL; // decimal point, so token type is REAL.
lexeme += current;
current = source.GetNextChar(); // consume decimal point
// collect fraction part of the number
var uval2 = UnsignedDigits();
err = uval2.Item1;
string frac_lexeme = uval2.Item2;
lexeme += frac_lexeme;
value = uval2.Item3;
frac_digits = uval2.Item4;
if (err == TokenType.ERROR)
{
TokenBuilder builder = new TokenBuilder();
return builder.CreateTokenWithType(err)
.WithLexeme(lexeme)
.AtPosition(pos)
.AtLine(line)
.WithValue(value)
.Build();
}
}
}
// is there an exponent part ?
// There cannot be an exponent if we already saw a ".." token.
current = source.GetCurrentChar();
char exponent_sign = '+';
string exp_digits = "";
if (!saw_dot_dot && (current == 'E' || current == 'e'))
{
type = TokenType.REAL;
lexeme += current;
current = source.GetNextChar(); // consume 'E' or 'e'
// exponent sign ?
if (current == '+' || current == '-')
{
lexeme += current;
exponent_sign = current;
current = source.GetNextChar();
}
// extract the digits of the exponent.
err = TokenType.INVALID;
var uval3 = UnsignedDigits();
err = uval3.Item1;
exp_digits = uval3.Item2;
value = uval3.Item3;
lexeme += exp_digits;
if (err == TokenType.ERROR)
{
TokenBuilder builder = new TokenBuilder();
return builder.CreateTokenWithType(err)
.WithLexeme(lexeme)
.AtPosition(pos)
.AtLine(line)
.WithValue(value)
.Build();
}
}
// compute the value of an integer number token
if (type == TokenType.INTEGER)
{
var uval4 = ComputeIntVal(whole_digits);
err = uval4.Item1;
int val = uval4.Item2;
value = uval4.Item3;
if (err == TokenType.ERROR)
{
TokenBuilder builder = new TokenBuilder();
return builder.CreateTokenWithType(err)
.WithLexeme(lexeme)
.AtPosition(pos)
.AtLine(line)
.WithValue(value)
.Build();
} else
{
value = val;
}
} else if (type == TokenType.REAL)
{
var dval1 = ComputeDoubleVal(whole_digits, frac_digits,
exp_digits, exponent_sign);
err = dval1.Item1;
double dval = dval1.Item2;
value = dval1.Item3;
if (err == TokenType.ERROR)
{
TokenBuilder builder = new TokenBuilder();
return builder.CreateTokenWithType(err)
.WithLexeme(lexeme)
.AtPosition(pos)
.AtLine(line)
.WithValue(value)
.Build();
} else
{
value = dval;
}
}
TokenBuilder builder0 = new TokenBuilder();
return builder0.CreateTokenWithType(err)
.WithLexeme(lexeme)
.AtPosition(pos)
.AtLine(line)
.WithValue(value)
.Build();
}