private Token ExtractToken() { SkipWhitespace(); char current = source.GetCurrentChar(); Token token; if (current == Source.END_OF_FILE) { TokenBuilder builder = new TokenBuilder(); token = builder.CreateTokenWithType(TokenType.END_OF_FILE) .AtLine(source.LineNumber) .AtPosition(source.Position) .Build(); } else if (Char.IsLetter(current)) { token = ExtractWordToken(); } else if (Char.IsDigit(current)) { token = ExtractNumberToken(); } else if (current == '\'') { token = ExtractStringToken(); } else if (TokenTypeInfo.IsSpecialSymbol(current.ToString())) { token = ExtractSpecialToken(); } else { TokenBuilder builder = new TokenBuilder(); token = builder.CreateTokenWithType(TokenType.ERROR) .WithLexeme(current.ToString()) .WithValue(ErrorCode.INVALID_CHARACTER) .AtLine(source.LineNumber) .AtPosition(source.Position) .Build(); source.GetNextChar(); } return token; }
private Token ExtractSpecialToken() { char current = source.GetCurrentChar(); StringBuilder lexeme = new StringBuilder(); lexeme.Append(current); int pos = source.Position; int line= source.LineNumber; string value = null; TokenType type = TokenType.INVALID; switch (current) { case '+': case '-': case '*': case '/': case ',': case ';': case '\'': case '=': case '(': case ')': case '[': case ']': case '{': case '}': case '^': source.GetNextChar(); break; case ':': current = source.GetNextChar(); // consume ':' if (current == '=') { lexeme.Append(current); source.GetNextChar(); // consume '=' } break; case '<': current = source.GetNextChar(); if (current == '=') { lexeme.Append(current); source.GetNextChar(); } else if (current == '>') { lexeme.Append(current); source.GetNextChar(); } break; case '>': current = source.GetNextChar(); if (current == '=') { lexeme.Append(current); source.GetNextChar(); } break; case '.': current = source.GetNextChar(); if (current == '.') { lexeme.Append(current); source.GetNextChar(); } break; default: source.GetNextChar(); type = TokenType.ERROR; value = ErrorCode.INVALID_CHARACTER.Message; break; } if (type == TokenType.INVALID) { type = TokenTypeInfo.SpecialTypeFromString(lexeme.ToString()); } TokenBuilder builder = new TokenBuilder(); return builder.CreateTokenWithType(type).WithLexeme(lexeme.ToString()) .WithValue(value) .AtPosition(pos) .AtLine(line).Build(); }
private Token ExtractStringToken() { int line = source.LineNumber; int pos = source.Position; StringBuilder lexeme = new StringBuilder(); StringBuilder value = new StringBuilder(); char current = source.GetNextChar(); // consume initial quote lexeme.Append('\''); // deal with string characters do { // replace any whitespace character with a blank if (Char.IsWhiteSpace(current)) { current = ' '; } if (current != '\'' && current != Source.END_OF_FILE) { lexeme.Append(current); value.Append(current); current = source.GetNextChar(); // again consume } // quote ? each pair of adjacent quotes represent a single quote. if (current == '\'') { while (current == '\'' && source.PeekNextChar() == '\'') { lexeme.Append("''"); value.Append('\''); current = source.GetNextChar(); current = source.GetNextChar(); // consume both quotes } } } while(current != '\'' && current != Source.END_OF_FILE); TokenBuilder builder = new TokenBuilder(); if (current == '\'') { source.GetNextChar(); // consume final quote lexeme.Append('\''); builder.CreateTokenWithType(TokenType.STRING).WithValue(value.ToString()).WithLexeme(lexeme.ToString()); } else { builder.CreateTokenWithType(TokenType.ERROR).WithValue(ErrorCode.UNEXPECTED_EOF); } return builder.AtPosition(pos).AtLine(line).Build(); }
private Token ExtractNumberToken() { int line = source.LineNumber; int pos = source.Position; var uval = UnsignedDigits(); TokenType err = uval.Item1; string lexeme = uval.Item2; object value = uval.Item3; string whole_digits = uval.Item4; if (err == TokenType.ERROR) { TokenBuilder builder = new TokenBuilder(); return builder.CreateTokenWithType(err) .WithLexeme(lexeme) .AtPosition(pos) .AtLine(line) .WithValue(value) .Build(); } // assume int TokenType type = TokenType.INTEGER; // Is there a . ? // It could be a decimal point or the start of a .. token char current = source.GetCurrentChar(); bool saw_dot_dot = false; string frac_digits = ""; if (current == '.') { if (source.PeekNextChar() == '.') { saw_dot_dot = true; // it's a ".." token, so don't consume it } else { type = TokenType.REAL; // decimal point, so token type is REAL. lexeme += current; current = source.GetNextChar(); // consume decimal point // collect fraction part of the number var uval2 = UnsignedDigits(); err = uval2.Item1; string frac_lexeme = uval2.Item2; lexeme += frac_lexeme; value = uval2.Item3; frac_digits = uval2.Item4; if (err == TokenType.ERROR) { TokenBuilder builder = new TokenBuilder(); return builder.CreateTokenWithType(err) .WithLexeme(lexeme) .AtPosition(pos) .AtLine(line) .WithValue(value) .Build(); } } } // is there an exponent part ? // There cannot be an exponent if we already saw a ".." token. current = source.GetCurrentChar(); char exponent_sign = '+'; string exp_digits = ""; if (!saw_dot_dot && (current == 'E' || current == 'e')) { type = TokenType.REAL; lexeme += current; current = source.GetNextChar(); // consume 'E' or 'e' // exponent sign ? if (current == '+' || current == '-') { lexeme += current; exponent_sign = current; current = source.GetNextChar(); } // extract the digits of the exponent. err = TokenType.INVALID; var uval3 = UnsignedDigits(); err = uval3.Item1; exp_digits = uval3.Item2; value = uval3.Item3; lexeme += exp_digits; if (err == TokenType.ERROR) { TokenBuilder builder = new TokenBuilder(); return builder.CreateTokenWithType(err) .WithLexeme(lexeme) .AtPosition(pos) .AtLine(line) .WithValue(value) .Build(); } } // compute the value of an integer number token if (type == TokenType.INTEGER) { var uval4 = ComputeIntVal(whole_digits); err = uval4.Item1; int val = uval4.Item2; value = uval4.Item3; if (err == TokenType.ERROR) { TokenBuilder builder = new TokenBuilder(); return builder.CreateTokenWithType(err) .WithLexeme(lexeme) .AtPosition(pos) .AtLine(line) .WithValue(value) .Build(); } else { value = val; } } else if (type == TokenType.REAL) { var dval1 = ComputeDoubleVal(whole_digits, frac_digits, exp_digits, exponent_sign); err = dval1.Item1; double dval = dval1.Item2; value = dval1.Item3; if (err == TokenType.ERROR) { TokenBuilder builder = new TokenBuilder(); return builder.CreateTokenWithType(err) .WithLexeme(lexeme) .AtPosition(pos) .AtLine(line) .WithValue(value) .Build(); } else { value = dval; } } TokenBuilder builder0 = new TokenBuilder(); return builder0.CreateTokenWithType(err) .WithLexeme(lexeme) .AtPosition(pos) .AtLine(line) .WithValue(value) .Build(); }