public void TextIterator_Simple()
        {
            CharacterStream ti = new CharacterStream(new StringTextProvider("abcd"));

            Assert.AreEqual(4, ti.TextProvider.Length);
            Assert.AreEqual(0, ti.Position);
            Assert.AreEqual('a', ti.CurrentChar);
            Assert.AreEqual(new DecodedChar('a', 1), TextHelpers.DecodeCurrentChar(ti));

            Assert.IsTrue(ti.TextProvider.CompareTo(ti.Position, "ab", ignoreCase: false));
            Assert.IsFalse(ti.TextProvider.CompareTo(ti.Position, "abcde", ignoreCase: false));

            Assert.IsTrue(TextHelpers.CompareCurrentDecodedString(ti, "ab", ignoreCase: false, matchLength: out int matchLength));
            Assert.AreEqual(2, matchLength);
            Assert.IsFalse(TextHelpers.CompareCurrentDecodedString(ti, "abcde", ignoreCase: false, matchLength: out _));

            Assert.IsFalse(ti.IsAtEnd);
            Assert.IsTrue(ti.Advance(1));
            Assert.AreEqual(1, ti.Position);
            Assert.AreEqual('b', ti.CurrentChar);
            Assert.AreEqual('a', ti.Peek(-1));
            Assert.AreEqual('c', ti.Peek(1));
            Assert.AreEqual('d', ti.Peek(2));
            Assert.AreEqual(0, ti.Peek(3));
            Assert.AreEqual(0, ti.Peek(4));

            Assert.IsTrue(ti.Advance(3));
            Assert.IsTrue(ti.IsAtEnd);

            Assert.IsFalse(ti.Advance(1));
        }
Esempio n. 2
0
        public void PeekWithOffsetAndReadCharacters()
        {
            string stringToRead = "quick";

            CharacterStream fcs = new CharacterStream(_reader);

            char[] charsPeeked = fcs.Peek(4, stringToRead.Length);

            Assert.IsTrue(
                charsPeeked != null &&
                charsPeeked.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsPeeked), stringToRead) == 0);

            stringToRead = "The quick";

            char[] charsRead = fcs.Read(stringToRead.Length);

            Assert.IsTrue(
                charsRead != null &&
                charsRead.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsRead), stringToRead) == 0);

            charsRead    = null;
            stringToRead = " fox jumped";

            charsRead = fcs.Read(stringToRead.Length);

            Assert.IsTrue(
                charsRead != null &&
                charsRead.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsRead), stringToRead) == 0);

            charsPeeked  = null;
            stringToRead = " over the lazy";

            charsPeeked = fcs.Peek(stringToRead.Length);

            Assert.IsTrue(
                charsPeeked != null &&
                charsPeeked.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsPeeked), stringToRead) == 0);
        }
Esempio n. 3
0
        public void PeekCharactersAtEndWithOffset()
        {
            CharacterStream fcs = new CharacterStream(_reader);

            char[] charsRead = fcs.Peek(TestString.Length - 3, 4);

            Assert.IsTrue(
                charsRead != null &&
                charsRead.Length == 3 &&
                string.CompareOrdinal(
                    new string(charsRead), "dog") == 0);

            charsRead = null;
            charsRead = fcs.Peek(5);

            Assert.IsTrue(
                charsRead != null &&
                charsRead.Length == 5 &&
                string.CompareOrdinal(
                    new string(charsRead), "The q") == 0);
        }
Esempio n. 4
0
        public void PeekCharactersWithOffset()
        {
            string stringToRead = "quick";

            CharacterStream fcs = new CharacterStream(_reader);

            char[] charsPeeked = fcs.Peek(4, stringToRead.Length);

            Assert.IsTrue(
                charsPeeked != null &&
                charsPeeked.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsPeeked), stringToRead) == 0);
        }
Esempio n. 5
0
        public void EndOfStream()
        {
            string          stringToRead = "The quick fox jumped over the ";
            CharacterStream fcs          = new CharacterStream(_reader);

            char[] charsRead = fcs.Read(stringToRead.Length);

            Assert.IsTrue(
                !fcs.EndOfSource &&
                charsRead != null &&
                charsRead.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsRead), stringToRead) == 0);

            stringToRead = "lazy brown dog";

            char[] charsPeeked = fcs.Peek(stringToRead.Length);

            Assert.IsTrue(
                !fcs.EndOfSource &&
                charsPeeked != null &&
                charsPeeked.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsPeeked), stringToRead) == 0);

            charsRead = fcs.Read(stringToRead.Length);

            Assert.IsTrue(
                !fcs.EndOfSource &&
                charsRead != null &&
                charsRead.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsRead), stringToRead) == 0);

            bool peekResult = fcs.Peek(out char pch);

            Assert.IsTrue(!peekResult && !fcs.EndOfSource && pch == '\0');

            bool readResult = fcs.Read(out char rch);

            Assert.IsTrue(!readResult && fcs.EndOfSource && rch == '\0');
        }
Esempio n. 6
0
        public void PeekCharactersAtEnd()
        {
            string stringToRead = "The quick fox jumped over the lazy brown ";

            CharacterStream fcs = new CharacterStream(_reader);

            char[] charsRead = fcs.Read(stringToRead.Length);

            Assert.IsTrue(
                charsRead != null &&
                charsRead.Length == stringToRead.Length &&
                string.CompareOrdinal(
                    new string(charsRead), stringToRead) == 0);

            charsRead = null;
            charsRead = fcs.Peek(4);

            Assert.IsTrue(
                charsRead != null &&
                charsRead.Length == 3 &&
                string.CompareOrdinal(
                    new string(charsRead), "dog") == 0);
        }
Esempio n. 7
0
        public Token NextToken()
        {
            if (peekedWords.Count > 0)
            {
                return(peekedWords.Dequeue());
            }

            Token res = new Token();

            char peekChar = stream.Peek();

            while (stream.PeekEquals("//"))
            {
                stream.DumpUntil('\n');
                stream.Next();
                peekChar = stream.Peek();
            }

            while (peekChar == ' ' || peekChar == '\n' || peekChar == '\r' || peekChar == '\t')
            {
                if (peekChar == '\n')
                {
                    TabIndex    = 0;
                    StartOfLine = true;
                }
                else if (peekChar == '\t' && StartOfLine)
                {
                    TabIndex++;
                }
                stream.Next();
                peekChar = stream.Peek();
            }

            res.row    = stream.Row;
            res.column = stream.Column;

            if (StartOfLine)
            {
                res.type    = TokenTypes.Punctuation_NewLine;
                res.value   = null;
                StartOfLine = false;
            }
            else if (peekChar == 0)
            {
                res.type  = TokenTypes.EOF;
                res.value = null;
            }
            else if (IsPunctuation(peekChar))
            {
                res.type  = punctuationDictionary[stream.Next()];
                res.value = null;
            }
            else if (char.IsDigit(peekChar))
            {
                res.type  = TokenTypes.Number;
                res.value = ReadNumber();
            }
            else if (peekChar == '"')
            {
                res.type  = TokenTypes.String;
                res.value = ReadString();
            }
            else if (IsOperatorStart(peekChar))
            {
                res.type  = operatorDictionary[ReadOperator()];
                res.value = null;
            }
            else if (IsKeyword(stream.PeekOnly(IsKeywordCharacters)))
            {
                res.type  = TokenTypes.Keyword;
                res.value = stream.NextOnly(IsKeywordCharacters);
            }
            else if (IsIdentifierStart(peekChar))
            {
                res.type  = TokenTypes.Identifier;
                res.value = stream.NextOnly(IsIdentifier);
            }
            else
            {
                throw new Exception("Unknown thing..");
            }

            return(res);
        }