public static List <TemplateToken> Tokenize(TokenizerStringReader StringReader) { List <TemplateToken> Tokens = new List <TemplateToken>(); Tokenize(Tokens, StringReader); return(Tokens); }
static public List<TemplateToken> Tokenize(List<TemplateToken> Tokens, TokenizerStringReader StringReader) { while (true) { var Match = StringReader.Match(StartTagRegex); if (Match.Success) { var RawText = StringReader.ReadString(Match.Index); if (RawText.Length > 0) Tokens.Add(new RawTemplateToken() { Text = RawText }); var OpenTagTokenString = StringReader.ReadString(2); // It is a comment. if (OpenTagTokenString == "{#") { int Index = StringReader.IndexOf("#}"); if (Index == -1) throw(new Exception("Not closing comment")); StringReader.ReadString(Index + 2); } // It is either a expression tag or a special tag. else { Tokens.Add(new OpenTagTemplateToken() { Text = OpenTagTokenString }); { ExpressionTokenizer.Tokenize(Tokens, StringReader); } var CloseTagTokenString = StringReader.ReadString(2); var ExpectedCloseTagTokenString = ((OpenTagTokenString == "{{") ? "}}" : "%}"); if (CloseTagTokenString != ExpectedCloseTagTokenString) { throw (new Exception("Expected '" + ExpectedCloseTagTokenString + "' but got '" + CloseTagTokenString + "'")); } Tokens.Add(new CloseTagTemplateToken() { Text = CloseTagTokenString }); continue; } } else { var RawText = StringReader.ReadString(); if (RawText.Length > 0) Tokens.Add(new RawTemplateToken() { Text = RawText }); return Tokens; } } }
public static void Tokenize(List<TemplateToken> Tokens, TokenizerStringReader StringReader) { //StringReader.SkipSpaces(); while (StringReader.Available > 0) { switch (StringReader.Peek(2)) { case "%}": case "}}": return; } var CharBase = StringReader.PeekChar(); switch (CharBase) { // Ignore Spaces case ' ': case '\t': case '\r': case '\n': case '\v': StringReader.Skip(); break; // String case '\'': case '"': { StringReader.SegmentSetStart(); StringReader.Skip(); while (true) { var Char = StringReader.ReadChar(); if (Char == '\\') { StringReader.ReadChar(); } else if (Char == CharBase) { break; } } StringReader.SegmentSetEnd(); Tokens.Add(new StringLiteralTemplateToken() { Text = StringReader.SegmentGetSlice(), }); } break; default: // Numbers if (IsDecimalDigit(CharBase)) { StringReader.SegmentSetStart(); StringReader.Skip(); while (true) { var Char = StringReader.PeekChar(); if (IsDecimalDigit(Char)) { StringReader.Skip(); } else { break; } } StringReader.SegmentSetEnd(); Tokens.Add(new NumericLiteralTemplateToken() { Text = StringReader.SegmentGetSlice(), }); } else { // Operators if (Operators2.Contains(StringReader.Peek(2))) { Tokens.Add(new OperatorTemplateToken() { Text = StringReader.ReadString(2), }); break; } if (Operators1.Contains(CharBase)) { Tokens.Add(new OperatorTemplateToken() { Text = StringReader.ReadChar().ToString(), }); break; } if (IsAlpha(CharBase)) { StringReader.SegmentSetStart(); StringReader.Skip(); while (true) { var Char = StringReader.PeekChar(); if (IsAlphaNumeric(Char)) { StringReader.Skip(); } else { break; } } StringReader.SegmentSetEnd(); Tokens.Add(new IdentifierTemplateToken() { Text = StringReader.SegmentGetSlice(), }); break; } StringReader.Skip(); throw (new Exception("Unknown Token Type : '" + CharBase + "'")); } break; } } }
public static List <TemplateToken> Tokenize(List <TemplateToken> Tokens, TokenizerStringReader StringReader) { while (true) { var Match = StringReader.Match(StartTagRegex); if (Match.Success) { var RawText = StringReader.ReadString(Match.Index); if (RawText.Length > 0) { Tokens.Add(new RawTemplateToken() { Text = RawText }); } var OpenTagTokenString = StringReader.ReadString(2); // It is a comment. if (OpenTagTokenString == "{#") { int Index = StringReader.IndexOf("#}"); if (Index == -1) { throw(new Exception("Not closing comment")); } StringReader.ReadString(Index + 2); } // It is either a expression tag or a special tag. else { Tokens.Add(new OpenTagTemplateToken() { Text = OpenTagTokenString }); { ExpressionTokenizer.Tokenize(Tokens, StringReader); } var CloseTagTokenString = StringReader.ReadString(2); var ExpectedCloseTagTokenString = ((OpenTagTokenString == "{{") ? "}}" : "%}"); if (CloseTagTokenString != ExpectedCloseTagTokenString) { throw (new Exception("Expected '" + ExpectedCloseTagTokenString + "' but got '" + CloseTagTokenString + "'")); } Tokens.Add(new CloseTagTemplateToken() { Text = CloseTagTokenString }); continue; } } else { var RawText = StringReader.ReadString(); if (RawText.Length > 0) { Tokens.Add(new RawTemplateToken() { Text = RawText }); } return(Tokens); } } }
static public void Tokenize(List <TemplateToken> Tokens, TokenizerStringReader StringReader) { //StringReader.SkipSpaces(); while (StringReader.Available > 0) { switch (StringReader.Peek(2)) { case "%}": case "}}": return; } var CharBase = StringReader.PeekChar(); switch (CharBase) { // Ignore Spaces case ' ': case '\t': case '\r': case '\n': case '\v': StringReader.Skip(); break; // String case '\'': case '"': { StringReader.SegmentSetStart(); StringReader.Skip(); while (true) { var Char = StringReader.ReadChar(); if (Char == '\\') { StringReader.ReadChar(); } else if (Char == CharBase) { break; } } StringReader.SegmentSetEnd(); Tokens.Add(new StringLiteralTemplateToken() { Text = StringReader.SegmentGetSlice(), }); } break; default: // Numbers if (IsDecimalDigit(CharBase)) { StringReader.SegmentSetStart(); StringReader.Skip(); while (true) { var Char = StringReader.PeekChar(); if (IsDecimalDigit(Char)) { StringReader.Skip(); } else { break; } } StringReader.SegmentSetEnd(); Tokens.Add(new NumericLiteralTemplateToken() { Text = StringReader.SegmentGetSlice(), }); } else { // Operators if (Operators2.Contains(StringReader.Peek(2))) { Tokens.Add(new OperatorTemplateToken() { Text = StringReader.ReadString(2), }); break; } if (Operators1.Contains(CharBase)) { Tokens.Add(new OperatorTemplateToken() { Text = StringReader.ReadChar().ToString(), }); break; } if (IsAlpha(CharBase)) { StringReader.SegmentSetStart(); StringReader.Skip(); while (true) { var Char = StringReader.PeekChar(); if (IsAlphaNumeric(Char)) { StringReader.Skip(); } else { break; } } StringReader.SegmentSetEnd(); Tokens.Add(new IdentifierTemplateToken() { Text = StringReader.SegmentGetSlice(), }); break; } StringReader.Skip(); throw (new Exception("Unknown Token Type : '" + CharBase + "'")); } break; } } }
static public List<TemplateToken> Tokenize(TokenizerStringReader StringReader) { List<TemplateToken> Tokens = new List<TemplateToken>(); Tokenize(Tokens, StringReader); return Tokens; }