public void Pop_IncrementsCurrentPositionByOne_WhenAllStreamElementsAreCharacters()
{
var stream = new StringStream("abs");
stream.Pop();
stream.Pop();
var result = stream.CurrentPosition;
Assert.AreEqual(2, result);
}
public void Pop_IncrementsCurrentPositionByLengthOfStreamElements_WhenStreamElementsAreLongerThanOne()
{
var stream = new StringStream();
stream.Append("as").Append("asd");
stream.Pop();
stream.Pop();
var result = stream.CurrentPosition;
Assert.AreEqual(5, result);
}
public void Pop_ReturnsFirstElement()
{
var stream = new StringStream("abs");
var result1 = stream.Pop();
var result2 = stream.Pop();
var result3 = stream.Pop();
Assert.AreEqual("a", result1);
Assert.AreEqual("b", result2);
Assert.AreEqual("s", result3);
}
public void Pop_NullString_ReturnsNullChar()
{
using (var scanner = new StringStream(null))
{
Assert.Equal('\0', scanner.Pop());
}
}
private Token ParseToken()
{
var TokType = TokenType.Error;
CurrentType_ = LexerStateType.Begin;
CurrentText_.Reset();
var TokenLine = CurrentLine_;
var TokenLineIndex = CurrentLineIndex_;
while (!CharsStream_.IsEnd() && CurrentType_ != LexerStateType.End)
{
var Ch = CharsStream_.Take();
CurrentText_.Push(Ch);
CurrentLineIndex_++;
switch (CurrentType_)
{
#region Begin
case LexerStateType.Begin:
if (LanguageDescriptor.IsWhitespaceChar(Ch))
{
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsEndOfLineChar(Ch))
{
CurrentLine_++;
CurrentLineIndex_ = 0;
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsDelimiterChar(Ch))
{
TokType = TokenType.Delimiter;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsDigitChar(Ch))
{
CurrentType_ = LexerStateType.Number;
}
else if (LanguageDescriptor.IsQuoteChar(Ch))
{
CurrentType_ = LexerStateType.String;
}
else if (LanguageDescriptor.IsIdentityChar(Ch))
{
CurrentType_ = LexerStateType.Identity;
}
break;
#endregion
#region Number
case LexerStateType.Number:
if (LanguageDescriptor.IsPointChar(Ch))
{
CurrentType_ = LexerStateType.Float;
}
else if (!LanguageDescriptor.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
LanguageDescriptor.IsDelimiterChar(Ch) ||
LanguageDescriptor.IsEndOfLineChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Float
case LexerStateType.Float:
if (!LanguageDescriptor.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
LanguageDescriptor.IsDelimiterChar(Ch) ||
LanguageDescriptor.IsEndOfLineChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region String
case LexerStateType.String:
if (LanguageDescriptor.IsEscapeChar(Ch))
{
CurrentText_.Pop();
CurrentText_.Push(LanguageDescriptor.EscapeChar(CharsStream_.Take()));
}
else if (LanguageDescriptor.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
{
CurrentText_.Pop();
CurrentText_.Remove(0);
TokType = TokenType.String;
CurrentType_ = LexerStateType.End;
}
else if (LanguageDescriptor.IsEndOfLineChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
TokType = TokenType.Error;
CurrentType_ = LexerStateType.End;
Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected <eof> in {CurrentText_}");
}
break;
#endregion
#region Identity
case LexerStateType.Identity:
if (!LanguageDescriptor.IsIdentityChar(Ch) && !LanguageDescriptor.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharsStream_.Back();
TokType = TokenType.Identity;
CurrentType_ = LexerStateType.End;
}
break;
#endregion
}
}
var TokCode = CurrentText_.ToString();
if (LanguageDescriptor.IsBooleanString(TokCode))
{
TokType = TokenType.Boolean;
}
else if (LanguageDescriptor.IsNullString(TokCode))
{
TokType = TokenType.Null;
}
return(new Token(TokType, TokCode, TokenLine, TokenLineIndex));
}
public void Index_MultilineString_CountsCorrectNumberOfChars()
{
const string input = @"Line one
Line two
Line three
Line Four";
using (var scanner = new StringStream(input))
{
long i;
for (i=0; !scanner.IsCompleted; i++)
{
scanner.Pop();
Assert.Equal(i, scanner.Index);
}
Assert.Equal(i-1, scanner.Index);
}
}
public void Column_MultilineString_CountsCorrectNumberOfColumns()
{
const string input = @"Line one
Line two
Line three
Line Four";
using (var scanner = new StringStream(input))
{
while (!scanner.IsCompleted)
{
scanner.Pop();
}
Assert.Equal(9, scanner.Column);
}
}
public void Peek_LongString_ReturnsSameAsPop()
{
const string input = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
using (var scanner = new StringStream(input))
{
while (!scanner.IsCompleted)
{
char ch = scanner.Peek();
Assert.Equal(scanner.Pop(), ch);
}
Assert.Equal(true, scanner.IsCompleted);
}
}
public void Pop_UnicodeString_ReturnsSameSequence()
{
const string input = "私が日本語を話すことはありません。";
using (var scanner = new StringStream(input))
{
var buffer = new StringBuilder();
while (!scanner.IsCompleted)
{
buffer.Append(scanner.Pop());
}
Assert.Equal(input, buffer.ToString());
}
}
public void Pop_EscapedSequence_ReturnsSameSequence()
{
const string input = @"""\\\b\f\n\r\t\u0123\u4567\u89AB\uCDEF\uabcd\uef4A\""""";
using (var scanner = new StringStream(input))
{
var buffer = new StringBuilder();
while (!scanner.IsCompleted)
{
buffer.Append(scanner.Pop());
}
Assert.Equal(input, buffer.ToString());
}
}
public void Pop_LongString_ReturnsSameSequence()
{
const string input = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
using (var scanner = new StringStream(input))
{
var buffer = new StringBuilder();
while (!scanner.IsCompleted)
{
buffer.Append(scanner.Pop());
}
Assert.Equal(input, buffer.ToString());
}
}
public void Pop_EmptyString_ReturnsEmptySequence()
{
const string input = "";
using (var scanner = new StringStream(input))
{
var buffer = new StringBuilder();
while (!scanner.IsCompleted)
{
buffer.Append(scanner.Pop());
}
Assert.Equal(input, buffer.ToString());
}
}
public void Pop_NullString_ReturnsEmptySequence()
{
using (var scanner = new StringStream(null))
{
var buffer = new StringBuilder();
while (!scanner.IsCompleted)
{
buffer.Append(scanner.Pop());
}
Assert.Equal(String.Empty, buffer.ToString());
}
}
private Token ParseToken()
{
var TokType = TokenType.Error;
CurrentType_ = LexerStateType.Begin;
CurrentText_.Reset();
while (!CharStream_.IsEnd() && CurrentType_ != LexerStateType.End)
{
var Ch = CharStream_.Take();
CurrentText_.Push(Ch);
switch (CurrentType_)
{
#region Begin
case LexerStateType.Begin:
if (LangDesc_.IsWhitespaceChar(Ch))
{
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsEndOfLineChar(Ch))
{
CurrentLine_++;
TokType = TokenType.None;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsDelimiterChar(Ch))
{
TokType = TokenType.Delimiter;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsDigitChar(Ch))
{
CurrentType_ = LexerStateType.Integer;
}
else if (LangDesc_.IsQuoteChar(Ch))
{
CurrentType_ = LexerStateType.String;
}
else if (LangDesc_.IsIdentityChar(Ch))
{
CurrentType_ = LexerStateType.Identity;
}
else if (LangDesc_.IsOperatorChar(Ch))
{
CurrentType_ = LexerStateType.Operator;
}
break;
#endregion
#region Integer
case LexerStateType.Integer:
if (Ch == '.')
{
CurrentType_ = LexerStateType.Float;
}
else if (!LangDesc_.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
if (LangDesc_.IsWhitespaceChar(Ch) ||
LangDesc_.IsOperatorChar(Ch) ||
LangDesc_.IsDelimiterChar(Ch) ||
LangDesc_.IsEndOfLineChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Float
case LexerStateType.Float:
if (!LangDesc_.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
if (LangDesc_.IsWhitespaceChar(Ch) || LangDesc_.IsDelimiterChar(Ch))
{
TokType = TokenType.Numeric;
}
else
{
TokType = TokenType.Error;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region String
case LexerStateType.String:
if (Ch == '\\')
{
CurrentText_.Pop();
CurrentText_.Push(CharStream_.Take());
}
else if (LangDesc_.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
{
CurrentText_.Pop();
CurrentText_.Remove(0);
TokType = TokenType.String;
CurrentType_ = LexerStateType.End;
}
else if (LangDesc_.IsEndOfLineChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
TokType = TokenType.Error;
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Identity
case LexerStateType.Identity:
if (!LangDesc_.IsIdentityChar(Ch) && !LangDesc_.IsDigitChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
var TokCode = CurrentText_.ToString();
if (LangDesc_.IsNilString(TokCode))
{
TokType = TokenType.Nil;
}
else if (LangDesc_.IsBooleanString(TokCode))
{
TokType = TokenType.Boolean;
}
else if (LangDesc_.IsKeywordString(TokCode))
{
TokType = TokenType.Keyword;
}
else
{
TokType = TokenType.Identifier;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
#region Operator
case LexerStateType.Operator:
if (!LangDesc_.IsOperatorChar(Ch))
{
CurrentText_.Pop();
CharStream_.Back();
if (LangDesc_.IsOperatorString(CurrentText_.ToString()))
{
TokType = TokenType.Operator;
}
else
{
TokType = TokenType.Error;
}
CurrentType_ = LexerStateType.End;
}
break;
#endregion
}
}
return(new Token(TokType, CurrentText_.ToString(), CurrentLine_));
}
