public static List <TemplateToken> Tokenize(TokenizerStringReader StringReader)
{
List <TemplateToken> Tokens = new List <TemplateToken>();
Tokenize(Tokens, StringReader);
return(Tokens);
}
static public List<TemplateToken> Tokenize(List<TemplateToken> Tokens, TokenizerStringReader StringReader)
{
while (true)
{
var Match = StringReader.Match(StartTagRegex);
if (Match.Success)
{
var RawText = StringReader.ReadString(Match.Index);
if (RawText.Length > 0) Tokens.Add(new RawTemplateToken() { Text = RawText });
var OpenTagTokenString = StringReader.ReadString(2);
// It is a comment.
if (OpenTagTokenString == "{#")
{
int Index = StringReader.IndexOf("#}");
if (Index == -1) throw(new Exception("Not closing comment"));
StringReader.ReadString(Index + 2);
}
// It is either a expression tag or a special tag.
else
{
Tokens.Add(new OpenTagTemplateToken() { Text = OpenTagTokenString });
{
ExpressionTokenizer.Tokenize(Tokens, StringReader);
}
var CloseTagTokenString = StringReader.ReadString(2);
var ExpectedCloseTagTokenString = ((OpenTagTokenString == "{{") ? "}}" : "%}");
if (CloseTagTokenString != ExpectedCloseTagTokenString)
{
throw (new Exception("Expected '" + ExpectedCloseTagTokenString + "' but got '" + CloseTagTokenString + "'"));
}
Tokens.Add(new CloseTagTemplateToken() { Text = CloseTagTokenString });
continue;
}
}
else
{
var RawText = StringReader.ReadString();
if (RawText.Length > 0) Tokens.Add(new RawTemplateToken() { Text = RawText });
return Tokens;
}
}
}
public static void Tokenize(List<TemplateToken> Tokens, TokenizerStringReader StringReader)
{
//StringReader.SkipSpaces();
while (StringReader.Available > 0)
{
switch (StringReader.Peek(2))
{
case "%}":
case "}}":
return;
}
var CharBase = StringReader.PeekChar();
switch (CharBase)
{
// Ignore Spaces
case ' ':
case '\t':
case '\r':
case '\n':
case '\v':
StringReader.Skip();
break;
// String
case '\'':
case '"':
{
StringReader.SegmentSetStart();
StringReader.Skip();
while (true)
{
var Char = StringReader.ReadChar();
if (Char == '\\')
{
StringReader.ReadChar();
}
else if (Char == CharBase)
{
break;
}
}
StringReader.SegmentSetEnd();
Tokens.Add(new StringLiteralTemplateToken()
{
Text = StringReader.SegmentGetSlice(),
});
}
break;
default:
// Numbers
if (IsDecimalDigit(CharBase))
{
StringReader.SegmentSetStart();
StringReader.Skip();
while (true)
{
var Char = StringReader.PeekChar();
if (IsDecimalDigit(Char))
{
StringReader.Skip();
}
else
{
break;
}
}
StringReader.SegmentSetEnd();
Tokens.Add(new NumericLiteralTemplateToken()
{
Text = StringReader.SegmentGetSlice(),
});
}
else
{
// Operators
if (Operators2.Contains(StringReader.Peek(2)))
{
Tokens.Add(new OperatorTemplateToken()
{
Text = StringReader.ReadString(2),
});
break;
}
if (Operators1.Contains(CharBase))
{
Tokens.Add(new OperatorTemplateToken()
{
Text = StringReader.ReadChar().ToString(),
});
break;
}
if (IsAlpha(CharBase))
{
StringReader.SegmentSetStart();
StringReader.Skip();
while (true)
{
var Char = StringReader.PeekChar();
if (IsAlphaNumeric(Char))
{
StringReader.Skip();
}
else
{
break;
}
}
StringReader.SegmentSetEnd();
Tokens.Add(new IdentifierTemplateToken()
{
Text = StringReader.SegmentGetSlice(),
});
break;
}
StringReader.Skip();
throw (new Exception("Unknown Token Type : '" + CharBase + "'"));
}
break;
}
}
}
public static List <TemplateToken> Tokenize(List <TemplateToken> Tokens, TokenizerStringReader StringReader)
{
while (true)
{
var Match = StringReader.Match(StartTagRegex);
if (Match.Success)
{
var RawText = StringReader.ReadString(Match.Index);
if (RawText.Length > 0)
{
Tokens.Add(new RawTemplateToken()
{
Text = RawText
});
}
var OpenTagTokenString = StringReader.ReadString(2);
// It is a comment.
if (OpenTagTokenString == "{#")
{
int Index = StringReader.IndexOf("#}");
if (Index == -1)
{
throw(new Exception("Not closing comment"));
}
StringReader.ReadString(Index + 2);
}
// It is either a expression tag or a special tag.
else
{
Tokens.Add(new OpenTagTemplateToken()
{
Text = OpenTagTokenString
});
{
ExpressionTokenizer.Tokenize(Tokens, StringReader);
}
var CloseTagTokenString = StringReader.ReadString(2);
var ExpectedCloseTagTokenString = ((OpenTagTokenString == "{{") ? "}}" : "%}");
if (CloseTagTokenString != ExpectedCloseTagTokenString)
{
throw (new Exception("Expected '" + ExpectedCloseTagTokenString + "' but got '" + CloseTagTokenString + "'"));
}
Tokens.Add(new CloseTagTemplateToken()
{
Text = CloseTagTokenString
});
continue;
}
}
else
{
var RawText = StringReader.ReadString();
if (RawText.Length > 0)
{
Tokens.Add(new RawTemplateToken()
{
Text = RawText
});
}
return(Tokens);
}
}
}
static public void Tokenize(List <TemplateToken> Tokens, TokenizerStringReader StringReader)
{
//StringReader.SkipSpaces();
while (StringReader.Available > 0)
{
switch (StringReader.Peek(2))
{
case "%}":
case "}}":
return;
}
var CharBase = StringReader.PeekChar();
switch (CharBase)
{
// Ignore Spaces
case ' ':
case '\t':
case '\r':
case '\n':
case '\v':
StringReader.Skip();
break;
// String
case '\'':
case '"':
{
StringReader.SegmentSetStart();
StringReader.Skip();
while (true)
{
var Char = StringReader.ReadChar();
if (Char == '\\')
{
StringReader.ReadChar();
}
else if (Char == CharBase)
{
break;
}
}
StringReader.SegmentSetEnd();
Tokens.Add(new StringLiteralTemplateToken()
{
Text = StringReader.SegmentGetSlice(),
});
}
break;
default:
// Numbers
if (IsDecimalDigit(CharBase))
{
StringReader.SegmentSetStart();
StringReader.Skip();
while (true)
{
var Char = StringReader.PeekChar();
if (IsDecimalDigit(Char))
{
StringReader.Skip();
}
else
{
break;
}
}
StringReader.SegmentSetEnd();
Tokens.Add(new NumericLiteralTemplateToken()
{
Text = StringReader.SegmentGetSlice(),
});
}
else
{
// Operators
if (Operators2.Contains(StringReader.Peek(2)))
{
Tokens.Add(new OperatorTemplateToken()
{
Text = StringReader.ReadString(2),
});
break;
}
if (Operators1.Contains(CharBase))
{
Tokens.Add(new OperatorTemplateToken()
{
Text = StringReader.ReadChar().ToString(),
});
break;
}
if (IsAlpha(CharBase))
{
StringReader.SegmentSetStart();
StringReader.Skip();
while (true)
{
var Char = StringReader.PeekChar();
if (IsAlphaNumeric(Char))
{
StringReader.Skip();
}
else
{
break;
}
}
StringReader.SegmentSetEnd();
Tokens.Add(new IdentifierTemplateToken()
{
Text = StringReader.SegmentGetSlice(),
});
break;
}
StringReader.Skip();
throw (new Exception("Unknown Token Type : '" + CharBase + "'"));
}
break;
}
}
}
static public List<TemplateToken> Tokenize(TokenizerStringReader StringReader)
{
List<TemplateToken> Tokens = new List<TemplateToken>();
Tokenize(Tokens, StringReader);
return Tokens;
}