public static List <TemplateToken> Tokenize(TokenizerStringReader StringReader)
        {
            List <TemplateToken> Tokens = new List <TemplateToken>();

            Tokenize(Tokens, StringReader);
            return(Tokens);
        }
Ejemplo n.º 2
0
		static public List<TemplateToken> Tokenize(List<TemplateToken> Tokens, TokenizerStringReader StringReader)
		{
			while (true)
			{
				var Match = StringReader.Match(StartTagRegex);

				if (Match.Success)
				{
					var RawText = StringReader.ReadString(Match.Index);
					if (RawText.Length > 0) Tokens.Add(new RawTemplateToken() { Text = RawText });
					var OpenTagTokenString = StringReader.ReadString(2);

					// It is a comment.
					if (OpenTagTokenString == "{#")
					{
						int Index = StringReader.IndexOf("#}");
						if (Index == -1) throw(new Exception("Not closing comment"));
						StringReader.ReadString(Index + 2);
					}
					// It is either a expression tag or a special tag.
					else
					{
						Tokens.Add(new OpenTagTemplateToken() { Text = OpenTagTokenString });
						{
							ExpressionTokenizer.Tokenize(Tokens, StringReader);
						}
						var CloseTagTokenString = StringReader.ReadString(2);
						var ExpectedCloseTagTokenString = ((OpenTagTokenString == "{{") ? "}}" : "%}");
						if (CloseTagTokenString != ExpectedCloseTagTokenString)
						{
							throw (new Exception("Expected '" + ExpectedCloseTagTokenString + "' but got '" + CloseTagTokenString + "'"));
						}
						Tokens.Add(new CloseTagTemplateToken() { Text = CloseTagTokenString });
						continue;
					}
				}
				else
				{
					var RawText = StringReader.ReadString();
					if (RawText.Length > 0) Tokens.Add(new RawTemplateToken() { Text = RawText });
					return Tokens;
				}
			}
		}
Ejemplo n.º 3
0
        public static void Tokenize(List<TemplateToken> Tokens, TokenizerStringReader StringReader)
        {
            //StringReader.SkipSpaces();
            while (StringReader.Available > 0)
            {
                switch (StringReader.Peek(2))
                {
                    case "%}":
                    case "}}":
                        return;
                }

                var CharBase = StringReader.PeekChar();

                switch (CharBase)
                {
                    // Ignore Spaces
                    case ' ':
                    case '\t':
                    case '\r':
                    case '\n':
                    case '\v':
                        StringReader.Skip();
                        break;
                    // String
                    case '\'':
                    case '"':
                        {
                            StringReader.SegmentSetStart();
                            StringReader.Skip();
                            while (true)
                            {
                                var Char = StringReader.ReadChar();
                                if (Char == '\\')
                                {
                                    StringReader.ReadChar();
                                }
                                else if (Char == CharBase)
                                {
                                    break;
                                }
                            }
                            StringReader.SegmentSetEnd();
                            Tokens.Add(new StringLiteralTemplateToken()
                            {
                                Text = StringReader.SegmentGetSlice(),
                            });
                        }
                        break;
                    default:
                        // Numbers
                        if (IsDecimalDigit(CharBase))
                        {
                            StringReader.SegmentSetStart();
                            StringReader.Skip();
                            while (true)
                            {
                                var Char = StringReader.PeekChar();
                                if (IsDecimalDigit(Char))
                                {
                                    StringReader.Skip();
                                }
                                else
                                {
                                    break;
                                }
                            }
                            StringReader.SegmentSetEnd();
                            Tokens.Add(new NumericLiteralTemplateToken()
                            {
                                Text = StringReader.SegmentGetSlice(),
                            });
                        }
                        else
                        {
                            // Operators
                            if (Operators2.Contains(StringReader.Peek(2)))
                            {
                                Tokens.Add(new OperatorTemplateToken()
                                {
                                    Text = StringReader.ReadString(2),
                                });
                                break;
                            }

                            if (Operators1.Contains(CharBase))
                            {
                                Tokens.Add(new OperatorTemplateToken()
                                {
                                    Text = StringReader.ReadChar().ToString(),
                                });
                                break;
                            }

                            if (IsAlpha(CharBase))
                            {
                                StringReader.SegmentSetStart();
                                StringReader.Skip();
                                while (true)
                                {
                                    var Char = StringReader.PeekChar();
                                    if (IsAlphaNumeric(Char))
                                    {
                                        StringReader.Skip();
                                    }
                                    else
                                    {
                                        break;
                                    }
                                }
                                StringReader.SegmentSetEnd();
                                Tokens.Add(new IdentifierTemplateToken()
                                {
                                    Text = StringReader.SegmentGetSlice(),
                                });
                                break;
                            }

                            StringReader.Skip();

                            throw (new Exception("Unknown Token Type : '" + CharBase + "'"));
                        }
                        break;
                }
            }
        }
        public static List <TemplateToken> Tokenize(List <TemplateToken> Tokens, TokenizerStringReader StringReader)
        {
            while (true)
            {
                var Match = StringReader.Match(StartTagRegex);

                if (Match.Success)
                {
                    var RawText = StringReader.ReadString(Match.Index);
                    if (RawText.Length > 0)
                    {
                        Tokens.Add(new RawTemplateToken()
                        {
                            Text = RawText
                        });
                    }
                    var OpenTagTokenString = StringReader.ReadString(2);

                    // It is a comment.
                    if (OpenTagTokenString == "{#")
                    {
                        int Index = StringReader.IndexOf("#}");
                        if (Index == -1)
                        {
                            throw(new Exception("Not closing comment"));
                        }
                        StringReader.ReadString(Index + 2);
                    }
                    // It is either a expression tag or a special tag.
                    else
                    {
                        Tokens.Add(new OpenTagTemplateToken()
                        {
                            Text = OpenTagTokenString
                        });
                        {
                            ExpressionTokenizer.Tokenize(Tokens, StringReader);
                        }
                        var CloseTagTokenString         = StringReader.ReadString(2);
                        var ExpectedCloseTagTokenString = ((OpenTagTokenString == "{{") ? "}}" : "%}");
                        if (CloseTagTokenString != ExpectedCloseTagTokenString)
                        {
                            throw (new Exception("Expected '" + ExpectedCloseTagTokenString + "' but got '" + CloseTagTokenString + "'"));
                        }
                        Tokens.Add(new CloseTagTemplateToken()
                        {
                            Text = CloseTagTokenString
                        });
                        continue;
                    }
                }
                else
                {
                    var RawText = StringReader.ReadString();
                    if (RawText.Length > 0)
                    {
                        Tokens.Add(new RawTemplateToken()
                        {
                            Text = RawText
                        });
                    }
                    return(Tokens);
                }
            }
        }
Ejemplo n.º 5
0
        static public void Tokenize(List <TemplateToken> Tokens, TokenizerStringReader StringReader)
        {
            //StringReader.SkipSpaces();
            while (StringReader.Available > 0)
            {
                switch (StringReader.Peek(2))
                {
                case "%}":
                case "}}":
                    return;
                }

                var CharBase = StringReader.PeekChar();

                switch (CharBase)
                {
                // Ignore Spaces
                case ' ':
                case '\t':
                case '\r':
                case '\n':
                case '\v':
                    StringReader.Skip();
                    break;

                // String
                case '\'':
                case '"':
                {
                    StringReader.SegmentSetStart();
                    StringReader.Skip();
                    while (true)
                    {
                        var Char = StringReader.ReadChar();
                        if (Char == '\\')
                        {
                            StringReader.ReadChar();
                        }
                        else if (Char == CharBase)
                        {
                            break;
                        }
                    }
                    StringReader.SegmentSetEnd();
                    Tokens.Add(new StringLiteralTemplateToken()
                        {
                            Text = StringReader.SegmentGetSlice(),
                        });
                }
                break;

                default:
                    // Numbers
                    if (IsDecimalDigit(CharBase))
                    {
                        StringReader.SegmentSetStart();
                        StringReader.Skip();
                        while (true)
                        {
                            var Char = StringReader.PeekChar();
                            if (IsDecimalDigit(Char))
                            {
                                StringReader.Skip();
                            }
                            else
                            {
                                break;
                            }
                        }
                        StringReader.SegmentSetEnd();
                        Tokens.Add(new NumericLiteralTemplateToken()
                        {
                            Text = StringReader.SegmentGetSlice(),
                        });
                    }
                    else
                    {
                        // Operators
                        if (Operators2.Contains(StringReader.Peek(2)))
                        {
                            Tokens.Add(new OperatorTemplateToken()
                            {
                                Text = StringReader.ReadString(2),
                            });
                            break;
                        }

                        if (Operators1.Contains(CharBase))
                        {
                            Tokens.Add(new OperatorTemplateToken()
                            {
                                Text = StringReader.ReadChar().ToString(),
                            });
                            break;
                        }

                        if (IsAlpha(CharBase))
                        {
                            StringReader.SegmentSetStart();
                            StringReader.Skip();
                            while (true)
                            {
                                var Char = StringReader.PeekChar();
                                if (IsAlphaNumeric(Char))
                                {
                                    StringReader.Skip();
                                }
                                else
                                {
                                    break;
                                }
                            }
                            StringReader.SegmentSetEnd();
                            Tokens.Add(new IdentifierTemplateToken()
                            {
                                Text = StringReader.SegmentGetSlice(),
                            });
                            break;
                        }

                        StringReader.Skip();

                        throw (new Exception("Unknown Token Type : '" + CharBase + "'"));
                    }
                    break;
                }
            }
        }
Ejemplo n.º 6
0
		static public List<TemplateToken> Tokenize(TokenizerStringReader StringReader)
		{
			List<TemplateToken> Tokens = new List<TemplateToken>();
			Tokenize(Tokens, StringReader);
			return Tokens;
		}