private Token ParseToken()
{
var TokType = TokenType.Error;
CurrentType_ = LexerStateType.Begin;
CurrentText_.Reset();
var TokenLine = CurrentLine_;
var TokenLineIndex = CurrentLineIndex_;
while (!CharsStream_.IsEnd() && CurrentType_ != LexerStateType.End)
{
var Ch = CharsStream_.Take();
CurrentText_.Push(Ch);
CurrentLineIndex_++;
switch (CurrentType_)
{
#region Begin
case LexerStateType.Begin:
if (LanguageDescriptor.IsWhitespaceChar(Ch))
{
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsEndOfLineChar(Ch))
{
CurrentLine_++;
CurrentLineIndex_ = 0;
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsDelimiterChar(Ch))
{
TokType = TokenType.Delimiter;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsDigitChar(Ch))
{
CurrentType_ = LexerStateType.Number;
}
else if (LanguageDescriptor.IsQuoteChar(Ch))
{
CurrentType_ = LexerStateType.String;
}
else if (LanguageDescriptor.IsIdentityChar(Ch))
{
CurrentType_ = LexerStateType.Identity;
}
break;
#endregion
#region Number
case LexerStateType.Number:
if (LanguageDescriptor.IsPointChar(Ch))
{
CurrentType_ = LexerStateType.Float;
}
else if (!LanguageDescriptor.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
LanguageDescriptor.IsDelimiterChar(Ch) ||
LanguageDescriptor.IsEndOfLineChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Float
case LexerStateType.Float:
if (!LanguageDescriptor.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
LanguageDescriptor.IsDelimiterChar(Ch) ||
LanguageDescriptor.IsEndOfLineChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region String
case LexerStateType.String:
if (LanguageDescriptor.IsEscapeChar(Ch))
{
CurrentText_.Pop();
CurrentText_.Push(LanguageDescriptor.EscapeChar(CharsStream_.Take()));
}
else if (LanguageDescriptor.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
{
CurrentText_.Pop();
CurrentText_.Remove(0);
TokType = TokenType.String;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsEndOfLineChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
TokType = TokenType.Error;
CurrentType_ = LexerStateType.End;
Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected <eof> in {CurrentText_}");
}
break;
#endregion
#region Identity
case LexerStateType.Identity:
if (!LanguageDescriptor.IsIdentityChar(Ch) && !LanguageDescriptor.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
TokType = TokenType.Identity;
CurrentType_ = LexerStateType.End;
}
break;
#endregion
}
}
var TokCode = CurrentText_.ToString();
if (LanguageDescriptor.IsBooleanString(TokCode))
{
TokType = TokenType.Boolean;
}
else if (LanguageDescriptor.IsNullString(TokCode))
{
TokType = TokenType.Null;
}
return(new Token(TokType, TokCode, TokenLine, TokenLineIndex));
}
private Token ParseToken()
{
var TokType = TokenType.Error;
CurrentType_ = LexerStateType.Begin;
CurrentText_.Reset();
while (!CharStream_.IsEnd() && CurrentType_ != LexerStateType.End)
{
var Ch = CharStream_.Take();
CurrentText_.Push(Ch);
switch (CurrentType_)
{
#region Begin
case LexerStateType.Begin:
if (LangDesc_.IsWhitespaceChar(Ch))
{
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsEndOfLineChar(Ch))
{
CurrentLine_++;
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsDelimiterChar(Ch))
{
TokType = TokenType.Delimiter;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsDigitChar(Ch))
{
CurrentType_ = LexerStateType.Integer;
}
else if (LangDesc_.IsQuoteChar(Ch))
{
CurrentType_ = LexerStateType.String;
}
else if (LangDesc_.IsIdentityChar(Ch))
{
CurrentType_ = LexerStateType.Identity;
}
else if (LangDesc_.IsOperatorChar(Ch))
{
CurrentType_ = LexerStateType.Operator;
}
break;
#endregion
#region Integer
case LexerStateType.Integer:
if (Ch == '.')
{
CurrentType_ = LexerStateType.Float;
}
else if (!LangDesc_.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
if (LangDesc_.IsWhitespaceChar(Ch) ||
LangDesc_.IsOperatorChar(Ch) ||
LangDesc_.IsDelimiterChar(Ch) ||
LangDesc_.IsEndOfLineChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Float
case LexerStateType.Float:
if (!LangDesc_.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
if (LangDesc_.IsWhitespaceChar(Ch) || LangDesc_.IsDelimiterChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region String
case LexerStateType.String:
if (Ch == '\\')
{
CurrentText_.Pop();
CurrentText_.Push(CharStream_.Take());
}
else if (LangDesc_.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
{
CurrentText_.Pop();
CurrentText_.Remove(0);
TokType = TokenType.String;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsEndOfLineChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
TokType = TokenType.Error;
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Identity
case LexerStateType.Identity:
if (!LangDesc_.IsIdentityChar(Ch) && !LangDesc_.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
var TokCode = CurrentText_.ToString();
if (LangDesc_.IsNilString(TokCode))
{
TokType = TokenType.Nil;
}
else if (LangDesc_.IsBooleanString(TokCode))
{
TokType = TokenType.Boolean;
}
else if (LangDesc_.IsKeywordString(TokCode))
{
TokType = TokenType.Keyword;
}
else
{
TokType = TokenType.Identifier;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Operator
case LexerStateType.Operator:
if (!LangDesc_.IsOperatorChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
if (LangDesc_.IsOperatorString(CurrentText_.ToString()))
{
TokType = TokenType.Operator;
}
else
{
TokType = TokenType.Error;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
}
}
return(new Token(TokType, CurrentText_.ToString(), CurrentLine_));
}