public void TextIterator_Simple()
{
CharacterStream ti = new CharacterStream(new StringTextProvider("abcd"));
Assert.AreEqual(4, ti.TextProvider.Length);
Assert.AreEqual(0, ti.Position);
Assert.AreEqual('a', ti.CurrentChar);
Assert.AreEqual(new DecodedChar('a', 1), TextHelpers.DecodeCurrentChar(ti));
Assert.IsTrue(ti.TextProvider.CompareTo(ti.Position, "ab", ignoreCase: false));
Assert.IsFalse(ti.TextProvider.CompareTo(ti.Position, "abcde", ignoreCase: false));
Assert.IsTrue(TextHelpers.CompareCurrentDecodedString(ti, "ab", ignoreCase: false, matchLength: out int matchLength));
Assert.AreEqual(2, matchLength);
Assert.IsFalse(TextHelpers.CompareCurrentDecodedString(ti, "abcde", ignoreCase: false, matchLength: out _));
Assert.IsFalse(ti.IsAtEnd);
Assert.IsTrue(ti.Advance(1));
Assert.AreEqual(1, ti.Position);
Assert.AreEqual('b', ti.CurrentChar);
Assert.AreEqual('a', ti.Peek(-1));
Assert.AreEqual('c', ti.Peek(1));
Assert.AreEqual('d', ti.Peek(2));
Assert.AreEqual(0, ti.Peek(3));
Assert.AreEqual(0, ti.Peek(4));
Assert.IsTrue(ti.Advance(3));
Assert.IsTrue(ti.IsAtEnd);
Assert.IsFalse(ti.Advance(1));
}
public void PeekWithOffsetAndReadCharacters()
{
string stringToRead = "quick";
CharacterStream fcs = new CharacterStream(_reader);
char[] charsPeeked = fcs.Peek(4, stringToRead.Length);
Assert.IsTrue(
charsPeeked != null &&
charsPeeked.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsPeeked), stringToRead) == 0);
stringToRead = "The quick";
char[] charsRead = fcs.Read(stringToRead.Length);
Assert.IsTrue(
charsRead != null &&
charsRead.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsRead), stringToRead) == 0);
charsRead = null;
stringToRead = " fox jumped";
charsRead = fcs.Read(stringToRead.Length);
Assert.IsTrue(
charsRead != null &&
charsRead.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsRead), stringToRead) == 0);
charsPeeked = null;
stringToRead = " over the lazy";
charsPeeked = fcs.Peek(stringToRead.Length);
Assert.IsTrue(
charsPeeked != null &&
charsPeeked.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsPeeked), stringToRead) == 0);
}
public void PeekCharactersAtEndWithOffset()
{
CharacterStream fcs = new CharacterStream(_reader);
char[] charsRead = fcs.Peek(TestString.Length - 3, 4);
Assert.IsTrue(
charsRead != null &&
charsRead.Length == 3 &&
string.CompareOrdinal(
new string(charsRead), "dog") == 0);
charsRead = null;
charsRead = fcs.Peek(5);
Assert.IsTrue(
charsRead != null &&
charsRead.Length == 5 &&
string.CompareOrdinal(
new string(charsRead), "The q") == 0);
}
public void PeekCharactersWithOffset()
{
string stringToRead = "quick";
CharacterStream fcs = new CharacterStream(_reader);
char[] charsPeeked = fcs.Peek(4, stringToRead.Length);
Assert.IsTrue(
charsPeeked != null &&
charsPeeked.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsPeeked), stringToRead) == 0);
}
public void EndOfStream()
{
string stringToRead = "The quick fox jumped over the ";
CharacterStream fcs = new CharacterStream(_reader);
char[] charsRead = fcs.Read(stringToRead.Length);
Assert.IsTrue(
!fcs.EndOfSource &&
charsRead != null &&
charsRead.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsRead), stringToRead) == 0);
stringToRead = "lazy brown dog";
char[] charsPeeked = fcs.Peek(stringToRead.Length);
Assert.IsTrue(
!fcs.EndOfSource &&
charsPeeked != null &&
charsPeeked.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsPeeked), stringToRead) == 0);
charsRead = fcs.Read(stringToRead.Length);
Assert.IsTrue(
!fcs.EndOfSource &&
charsRead != null &&
charsRead.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsRead), stringToRead) == 0);
bool peekResult = fcs.Peek(out char pch);
Assert.IsTrue(!peekResult && !fcs.EndOfSource && pch == '\0');
bool readResult = fcs.Read(out char rch);
Assert.IsTrue(!readResult && fcs.EndOfSource && rch == '\0');
}
public void PeekCharactersAtEnd()
{
string stringToRead = "The quick fox jumped over the lazy brown ";
CharacterStream fcs = new CharacterStream(_reader);
char[] charsRead = fcs.Read(stringToRead.Length);
Assert.IsTrue(
charsRead != null &&
charsRead.Length == stringToRead.Length &&
string.CompareOrdinal(
new string(charsRead), stringToRead) == 0);
charsRead = null;
charsRead = fcs.Peek(4);
Assert.IsTrue(
charsRead != null &&
charsRead.Length == 3 &&
string.CompareOrdinal(
new string(charsRead), "dog") == 0);
}
public Token NextToken()
{
if (peekedWords.Count > 0)
{
return(peekedWords.Dequeue());
}
Token res = new Token();
char peekChar = stream.Peek();
while (stream.PeekEquals("//"))
{
stream.DumpUntil('\n');
stream.Next();
peekChar = stream.Peek();
}
while (peekChar == ' ' || peekChar == '\n' || peekChar == '\r' || peekChar == '\t')
{
if (peekChar == '\n')
{
TabIndex = 0;
StartOfLine = true;
}
else if (peekChar == '\t' && StartOfLine)
{
TabIndex++;
}
stream.Next();
peekChar = stream.Peek();
}
res.row = stream.Row;
res.column = stream.Column;
if (StartOfLine)
{
res.type = TokenTypes.Punctuation_NewLine;
res.value = null;
StartOfLine = false;
}
else if (peekChar == 0)
{
res.type = TokenTypes.EOF;
res.value = null;
}
else if (IsPunctuation(peekChar))
{
res.type = punctuationDictionary[stream.Next()];
res.value = null;
}
else if (char.IsDigit(peekChar))
{
res.type = TokenTypes.Number;
res.value = ReadNumber();
}
else if (peekChar == '"')
{
res.type = TokenTypes.String;
res.value = ReadString();
}
else if (IsOperatorStart(peekChar))
{
res.type = operatorDictionary[ReadOperator()];
res.value = null;
}
else if (IsKeyword(stream.PeekOnly(IsKeywordCharacters)))
{
res.type = TokenTypes.Keyword;
res.value = stream.NextOnly(IsKeywordCharacters);
}
else if (IsIdentifierStart(peekChar))
{
res.type = TokenTypes.Identifier;
res.value = stream.NextOnly(IsIdentifier);
}
else
{
throw new Exception("Unknown thing..");
}
return(res);
}