Beispiel #1
0
        public void Pop_IncrementsCurrentPositionByOne_WhenAllStreamElementsAreCharacters()
        {
            var stream = new StringStream("abs");

            stream.Pop();
            stream.Pop();
            var result = stream.CurrentPosition;

            Assert.AreEqual(2, result);
        }
Beispiel #2
0
        public void Pop_IncrementsCurrentPositionByLengthOfStreamElements_WhenStreamElementsAreLongerThanOne()
        {
            var stream = new StringStream();

            stream.Append("as").Append("asd");

            stream.Pop();
            stream.Pop();
            var result = stream.CurrentPosition;

            Assert.AreEqual(5, result);
        }
Beispiel #3
0
        public void Pop_ReturnsFirstElement()
        {
            var stream = new StringStream("abs");

            var result1 = stream.Pop();
            var result2 = stream.Pop();
            var result3 = stream.Pop();

            Assert.AreEqual("a", result1);
            Assert.AreEqual("b", result2);
            Assert.AreEqual("s", result3);
        }
		public void Pop_NullString_ReturnsNullChar()
		{
			using (var scanner = new StringStream(null))
			{
				Assert.Equal('\0', scanner.Pop());
			}
		}
Beispiel #5
0
        private Token ParseToken()
        {
            var TokType = TokenType.Error;

            CurrentType_ = LexerStateType.Begin;
            CurrentText_.Reset();

            var TokenLine      = CurrentLine_;
            var TokenLineIndex = CurrentLineIndex_;

            while (!CharsStream_.IsEnd() && CurrentType_ != LexerStateType.End)
            {
                var Ch = CharsStream_.Take();
                CurrentText_.Push(Ch);
                CurrentLineIndex_++;

                switch (CurrentType_)
                {
                    #region Begin
                case LexerStateType.Begin:
                    if (LanguageDescriptor.IsWhitespaceChar(Ch))
                    {
                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsEndOfLineChar(Ch))
                    {
                        CurrentLine_++;
                        CurrentLineIndex_ = 0;

                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsDelimiterChar(Ch))
                    {
                        TokType      = TokenType.Delimiter;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Number;
                    }
                    else if (LanguageDescriptor.IsQuoteChar(Ch))
                    {
                        CurrentType_ = LexerStateType.String;
                    }
                    else if (LanguageDescriptor.IsIdentityChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Identity;
                    }
                    break;

                    #endregion
                    #region Number
                case LexerStateType.Number:
                    if (LanguageDescriptor.IsPointChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Float;
                    }
                    else if (!LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();

                        if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
                            LanguageDescriptor.IsDelimiterChar(Ch) ||
                            LanguageDescriptor.IsEndOfLineChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                            Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Float
                case LexerStateType.Float:
                    if (!LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();

                        if (LanguageDescriptor.IsWhitespaceChar(Ch) ||
                            LanguageDescriptor.IsDelimiterChar(Ch) ||
                            LanguageDescriptor.IsEndOfLineChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                            Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected character '{Ch}' in {CurrentText_}");
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region String
                case LexerStateType.String:
                    if (LanguageDescriptor.IsEscapeChar(Ch))
                    {
                        CurrentText_.Pop();
                        CurrentText_.Push(LanguageDescriptor.EscapeChar(CharsStream_.Take()));
                    }
                    else if (LanguageDescriptor.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
                    {
                        CurrentText_.Pop();
                        CurrentText_.Remove(0);

                        TokType      = TokenType.String;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LanguageDescriptor.IsEndOfLineChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();

                        TokType      = TokenType.Error;
                        CurrentType_ = LexerStateType.End;
                        Logger.Add(CurrentLine_, CurrentLineIndex_, $"unexpected <eof> in {CurrentText_}");
                    }
                    break;

                    #endregion
                    #region Identity
                case LexerStateType.Identity:
                    if (!LanguageDescriptor.IsIdentityChar(Ch) && !LanguageDescriptor.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharsStream_.Back();
                        TokType      = TokenType.Identity;
                        CurrentType_ = LexerStateType.End;
                    }
                    break;
                    #endregion
                }
            }

            var TokCode = CurrentText_.ToString();
            if (LanguageDescriptor.IsBooleanString(TokCode))
            {
                TokType = TokenType.Boolean;
            }
            else if (LanguageDescriptor.IsNullString(TokCode))
            {
                TokType = TokenType.Null;
            }

            return(new Token(TokType, TokCode, TokenLine, TokenLineIndex));
        }
		public void Index_MultilineString_CountsCorrectNumberOfChars()
		{
			const string input = @"Line one
Line two
Line three
Line Four";

			using (var scanner = new StringStream(input))
			{
				long i;
				for (i=0; !scanner.IsCompleted; i++)
				{
					scanner.Pop();
					Assert.Equal(i, scanner.Index);
				}

				Assert.Equal(i-1, scanner.Index);
			}
		}
		public void Column_MultilineString_CountsCorrectNumberOfColumns()
		{
			const string input = @"Line one
Line two
Line three
Line Four";

			using (var scanner = new StringStream(input))
			{
				while (!scanner.IsCompleted)
				{
					scanner.Pop();
				}

				Assert.Equal(9, scanner.Column);
			}
		}
		public void Peek_LongString_ReturnsSameAsPop()
		{
			const string input = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";

			using (var scanner = new StringStream(input))
			{
				while (!scanner.IsCompleted)
				{
					char ch = scanner.Peek();
					Assert.Equal(scanner.Pop(), ch);
				}

				Assert.Equal(true, scanner.IsCompleted);
			}
		}
		public void Pop_UnicodeString_ReturnsSameSequence()
		{
			const string input = "私が日本語を話すことはありません。";

			using (var scanner = new StringStream(input))
			{
				var buffer = new StringBuilder();
				while (!scanner.IsCompleted)
				{
					buffer.Append(scanner.Pop());
				}

				Assert.Equal(input, buffer.ToString());
			}
		}
		public void Pop_EscapedSequence_ReturnsSameSequence()
		{
			const string input = @"""\\\b\f\n\r\t\u0123\u4567\u89AB\uCDEF\uabcd\uef4A\""""";

			using (var scanner = new StringStream(input))
			{
				var buffer = new StringBuilder();
				while (!scanner.IsCompleted)
				{
					buffer.Append(scanner.Pop());
				}

				Assert.Equal(input, buffer.ToString());
			}
		}
		public void Pop_LongString_ReturnsSameSequence()
		{
			const string input = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";

			using (var scanner = new StringStream(input))
			{
				var buffer = new StringBuilder();
				while (!scanner.IsCompleted)
				{
					buffer.Append(scanner.Pop());
				}

				Assert.Equal(input, buffer.ToString());
			}
		}
		public void Pop_EmptyString_ReturnsEmptySequence()
		{
			const string input = "";

			using (var scanner = new StringStream(input))
			{
				var buffer = new StringBuilder();
				while (!scanner.IsCompleted)
				{
					buffer.Append(scanner.Pop());
				}

				Assert.Equal(input, buffer.ToString());
			}
		}
		public void Pop_NullString_ReturnsEmptySequence()
		{
			using (var scanner = new StringStream(null))
			{
				var buffer = new StringBuilder();
				while (!scanner.IsCompleted)
				{
					buffer.Append(scanner.Pop());
				}

				Assert.Equal(String.Empty, buffer.ToString());
			}
		}
Beispiel #14
0
        private Token ParseToken()
        {
            var TokType = TokenType.Error;

            CurrentType_ = LexerStateType.Begin;
            CurrentText_.Reset();

            while (!CharStream_.IsEnd() && CurrentType_ != LexerStateType.End)
            {
                var Ch = CharStream_.Take();
                CurrentText_.Push(Ch);

                switch (CurrentType_)
                {
                    #region Begin
                case LexerStateType.Begin:
                    if (LangDesc_.IsWhitespaceChar(Ch))
                    {
                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsEndOfLineChar(Ch))
                    {
                        CurrentLine_++;

                        TokType      = TokenType.None;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsDelimiterChar(Ch))
                    {
                        TokType      = TokenType.Delimiter;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Integer;
                    }
                    else if (LangDesc_.IsQuoteChar(Ch))
                    {
                        CurrentType_ = LexerStateType.String;
                    }
                    else if (LangDesc_.IsIdentityChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Identity;
                    }
                    else if (LangDesc_.IsOperatorChar(Ch))
                    {
                        CurrentType_ = LexerStateType.Operator;
                    }
                    break;

                    #endregion
                    #region Integer
                case LexerStateType.Integer:
                    if (Ch == '.')
                    {
                        CurrentType_ = LexerStateType.Float;
                    }
                    else if (!LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        if (LangDesc_.IsWhitespaceChar(Ch) ||
                            LangDesc_.IsOperatorChar(Ch) ||
                            LangDesc_.IsDelimiterChar(Ch) ||
                            LangDesc_.IsEndOfLineChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Float
                case LexerStateType.Float:
                    if (!LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        if (LangDesc_.IsWhitespaceChar(Ch) || LangDesc_.IsDelimiterChar(Ch))
                        {
                            TokType = TokenType.Numeric;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region String
                case LexerStateType.String:
                    if (Ch == '\\')
                    {
                        CurrentText_.Pop();
                        CurrentText_.Push(CharStream_.Take());
                    }
                    else if (LangDesc_.IsQuoteChar(Ch) && CurrentText_.Index(0) == Ch)
                    {
                        CurrentText_.Pop();
                        CurrentText_.Remove(0);

                        TokType      = TokenType.String;
                        CurrentType_ = LexerStateType.End;
                    }
                    else if (LangDesc_.IsEndOfLineChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        TokType      = TokenType.Error;
                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Identity
                case LexerStateType.Identity:
                    if (!LangDesc_.IsIdentityChar(Ch) && !LangDesc_.IsDigitChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        var TokCode = CurrentText_.ToString();
                        if (LangDesc_.IsNilString(TokCode))
                        {
                            TokType = TokenType.Nil;
                        }
                        else if (LangDesc_.IsBooleanString(TokCode))
                        {
                            TokType = TokenType.Boolean;
                        }
                        else if (LangDesc_.IsKeywordString(TokCode))
                        {
                            TokType = TokenType.Keyword;
                        }
                        else
                        {
                            TokType = TokenType.Identifier;
                        }
                        CurrentType_ = LexerStateType.End;
                    }
                    break;

                    #endregion
                    #region Operator
                case LexerStateType.Operator:
                    if (!LangDesc_.IsOperatorChar(Ch))
                    {
                        CurrentText_.Pop();
                        CharStream_.Back();

                        if (LangDesc_.IsOperatorString(CurrentText_.ToString()))
                        {
                            TokType = TokenType.Operator;
                        }
                        else
                        {
                            TokType = TokenType.Error;
                        }

                        CurrentType_ = LexerStateType.End;
                    }
                    break;
                    #endregion
                }
            }

            return(new Token(TokType, CurrentText_.ToString(), CurrentLine_));
        }