internal TextSpan (Token start, Token end)
{
this.startLine = start.StartLine;
this.startColumn = start.StartColumn;
this.endLine = end.StartLine;
this.endColumn = end.StartColumn + end.Width;
this.startPosition = end.StartPosition + end.Width;
this.endPosition = EndPosition;
}
public IdentifierToken(Identifier Spelling, int Width, Token.Type Kind, int StartCharacterPosition, int StartLine, int StartColumn, bool FirstOnLine)
:base(Kind, StartCharacterPosition, StartLine, StartColumn, FirstOnLine)
{
this.Spelling = Spelling;
this.width = Width;
}
public Token(Token.Type Kind, int StartPosition, int StartLine, int StartColumn, bool FirstOnLine)
{
this.Kind = Kind;
this.StartPosition = StartPosition;
this.StartLine = StartLine;
this.StartColumn = StartColumn;
this.firstOnLine = FirstOnLine;
}
public Token InsertSemicolonBefore()
{
Token semicolon = new Token (Token.Type.Semicolon, StartPosition, StartLine, StartColumn, false);
semicolon.next = this;
return semicolon;
}
private bool CheckSyntaxExpected (Token.Type type)
{
if (current.Kind == type)
return true;
//default
DiagnosticCode code = DiagnosticCode.SyntaxError;
switch (type) {
case Token.Type.@case:
case Token.Type.@default:
code = DiagnosticCode.CaseOrDefaultExpected;
break;
case Token.Type.Identifier:
code = DiagnosticCode.IdentifierExpected;
break;
case Token.Type.LeftBrace:
code = DiagnosticCode.LeftBraceExpected;
break;
case Token.Type.LeftParenthesis:
code = DiagnosticCode.LeftParenExpected;
break;
case Token.Type.Semicolon:
code = DiagnosticCode.SemicolonExpected;
break;
}
if (current.Kind == Token.Type.EndOfInput)
syntaxIncomplete = true;
Error (code, new TextSpan (current,current));
return false;
}
private TokenCategory GetCategory(Token.Type type)
{
switch (type) {
case Token.Type.None:
case Token.Type.final:
case Token.Type.OctalIntegerLiteral:
case Token.Type.HexIntegerLiteral:
return TokenCategory.None;
case Token.Type.EndOfInput:
return TokenCategory.EndOfStream;
case Token.Type.LeftBrace:
case Token.Type.RightBrace:
case Token.Type.LeftParenthesis:
case Token.Type.RightParenthesis:
case Token.Type.LeftBracket:
case Token.Type.RightBracket:
case Token.Type.Dot:
case Token.Type.Semicolon:
case Token.Type.Comma:
return TokenCategory.Delimiter;
case Token.Type.Less:
case Token.Type.Greater:
case Token.Type.LessEqual:
case Token.Type.GreaterEqual:
case Token.Type.EqualEqual:
case Token.Type.BangEqual:
case Token.Type.EqualEqualEqual:
case Token.Type.BangEqualEqual:
case Token.Type.Plus:
case Token.Type.Minus:
case Token.Type.Star:
case Token.Type.Percent:
case Token.Type.PlusPlus:
case Token.Type.MinusMinus:
case Token.Type.LessLess:
case Token.Type.GreaterGreater:
case Token.Type.GreaterGreaterGreater:
case Token.Type.Ampersand:
case Token.Type.Bar:
case Token.Type.Circumflex:
case Token.Type.Bang:
case Token.Type.Tilda:
case Token.Type.AmpersandAmpersand:
case Token.Type.BarBar:
case Token.Type.Question:
case Token.Type.Colon:
case Token.Type.Equal:
case Token.Type.PlusEqual:
case Token.Type.MinusEqual:
case Token.Type.StarEqual:
case Token.Type.PercentEqual:
case Token.Type.LessLessEqual:
case Token.Type.GreaterGreaterEqual:
case Token.Type.GreaterGreaterGreaterEqual:
case Token.Type.AmpersandEqual:
case Token.Type.BarEqual:
case Token.Type.CircumflexEqual:
case Token.Type.Divide:
case Token.Type.DivideEqual:
return TokenCategory.Operator;
case Token.Type.@break:
case Token.Type.@else:
case Token.Type.@new:
case Token.Type.var:
case Token.Type.@case:
case Token.Type.@finally:
case Token.Type.@return:
case Token.Type.@void:
case Token.Type.@catch:
case Token.Type.@for:
case Token.Type.@switch:
case Token.Type.@while:
case Token.Type.@continue:
case Token.Type.function:
case Token.Type.@this:
case Token.Type.with:
case Token.Type.@default:
case Token.Type.@if:
case Token.Type.@throw:
case Token.Type.delete:
case Token.Type.@in:
case Token.Type.@try:
case Token.Type.@do:
case Token.Type.instanceof:
case Token.Type.@typeof:
case Token.Type.@abstract:
case Token.Type.@enum:
case Token.Type.@int:
case Token.Type.@short:
case Token.Type.boolean:
case Token.Type.export:
case Token.Type.@interface:
case Token.Type.@static:
case Token.Type.@byte:
case Token.Type.extends:
case Token.Type.@long:
case Token.Type.super:
case Token.Type.@char:
case Token.Type.native:
case Token.Type.synchronized:
case Token.Type.@class:
case Token.Type.@float:
case Token.Type.packate:
case Token.Type.throws:
case Token.Type.@const:
case Token.Type.@goto:
case Token.Type.@private:
case Token.Type.transient:
case Token.Type.debugger:
case Token.Type.implements:
case Token.Type.@protected:
case Token.Type.@volatile:
case Token.Type.@double:
case Token.Type.import:
case Token.Type.@public:
case Token.Type.@null:
case Token.Type.@true:
case Token.Type.@false:
return TokenCategory.Keyword;
case Token.Type.NumericLiteral:
return TokenCategory.NumericLiteral;
case Token.Type.StringLiteral:
return TokenCategory.StringLiteral;
case Token.Type.RegularExpressionLiteral:
return TokenCategory.RegularExpressionLiteral;
case Token.Type.Identifier:
return TokenCategory.Identifier;
case Token.Type.Bad:
return TokenCategory.Error;
}
return TokenCategory.None;
}
private Token CreateHexIntegerLiteralToken (int first)
{
StringBuilder builder = new StringBuilder ();
builder.Append ((char)first);//0
builder.Append ((char)ReadChar ()); //x or X (ever tested before)
double val = 0;
while (Advance ()) {
int next = PeekChar ();
switch (next) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
builder.Append ((char)next);
val = val * 16 + next - '0';
ReadChar ();
continue;
}
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f': {
builder.Append ((char)next);
val = val * 16 + next - 'a' + 10;
ReadChar ();
continue;
}
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F': {
builder.Append ((char)next);
val = val * 16 + next - 'A' + 10;
ReadChar ();
continue;
}
}
break;
}
string result = builder.ToString ();
current = new HexIntegerLiteralToken (result, val, position, Line, Column, FirstOnLine);
return current;
}
private void Next ()
{
current = lexer.GetNext ();
}
private Token CreateOctalIntegerLiteralToken (int first)
{
StringBuilder builder = new StringBuilder ();
builder.Append ((char)first);//0
double val = 0;
while (Advance ()) {
int next = PeekChar ();
switch (next) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7': {
builder.Append ((char)next);
val = val * 8 + next - '0';
ReadChar ();
continue;
}
}
break;
}
string result = builder.ToString ();
current = new OctalIntegerLiteralToken (result, val ,position, Line, Column, FirstOnLine);
return current;
}
private Token CreateNumericLiteralToken (int first)
{
int next = PeekChar ();
if (first == '0') {
if (next == 'x' || next == 'X')
return CreateHexIntegerLiteralToken (first);
else {
switch (next) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
return CreateOctalIntegerLiteralToken (first);
}//else continue to make a numerical literal token
}
}
StringBuilder builder = new StringBuilder ();
builder.Append ((char) first);
int dot = 0;
int exp = 0;
while (Advance ()) {
next = PeekChar ();
switch (next) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
builder.Append ((char) next);
ReadChar ();
continue;
}
case '.': {
ReadChar ();
dot++;
builder.Append ((char)next);
if (dot > 1) {
current = CreateBadToken (builder.ToString(), DiagnosticCode.MalformedNumericLiteral);
return current;
}
continue;
}
case 'e':
case 'E': {
builder.Append ((char)next);
ReadChar ();
exp++;
if (exp > 1) {
current = CreateBadToken (builder.ToString (), DiagnosticCode.MalformedNumericLiteral);
return current;
}
next = PeekChar ();
if (next == '+' || next == '-') {
builder.Append ((char)next);
ReadChar ();
}
continue;
}
}
break;
}
string result = builder.ToString ();
current = new NumericLiteralToken (result, position, Line, Column, FirstOnLine);
return current;
}
private Token CreateIdentOrKeyword (int first)
{
StringBuilder builder = new StringBuilder ();
builder.Append ((char) first);
while (Advance ()) {
int next = PeekChar ();
if (Char.IsLetterOrDigit ((char) next) || next == '$' || next == '_') {
builder.Append ((char) next);
ReadChar ();
continue;
}
break;
}
string result = builder.ToString ();
Token.Type type;
bool keyword = keywords.TryGetValue (result, out type);
if (keyword) {
current = new Token (type, position, Line, Column, FirstOnLine);
} else {
this.IDTable.InsertIdentifier (result);
Identifier id = new Identifier (result, IDTable);
current = new IdentifierToken (id, result.Length, Token.Type.Identifier, position, Line,
Column, FirstOnLine);
}
return current;
}
private Token CreateToken (Token.Type tokenType, string value)
{
current = new Token (tokenType, position, Line, Column, FirstOnLine);
return current;
}
public RegularExpressionLiteralToken ScanRegularExpression (Token Divide)
{
int startpos = Divide.StartPosition;
int startrow = Divide.StartLine;
int startcol = Divide.StartColumn;
Token current = Divide[this];
StringBuilder regexp = new StringBuilder ();
while (current.Kind == Token.Type.Divide) {
//get the value of the current token
for (int i = 0; i < current.Width; i++)
regexp.Append(source [position + i]);
current = Divide[this];
}
//TODO flags maybe but not always so must peek and not advance
//TODO firstonline
return new RegularExpressionLiteralToken (regexp.ToString (), "", position - startpos, startpos, startrow, startcol, false);
}