bool IEnumerator.MoveNext()
{
if (CurrentIndex >= Tokens.Count)
{
return(false);
}
_current = Tokens[CurrentIndex];
CurrentIndex++;
return(true);
}
/// <summary>
/// Goes through the template and evaluates all tokens that are enclosed by {{ }}.
/// </summary>
/// <param name="parserOptions"></param>
/// <param name="context"></param>
/// <returns></returns>
public static async TokenizerResultPromise Tokenize(ParserOptions parserOptions,
TokenzierContext context)
{
var templateString = parserOptions.Template;
var scopestack = new Stack <ScopeStackItem>();
var partialsNames = new List <string>(parserOptions.PartialsStore?.GetNames() ?? new string[0]);
context.SetLocation(0);
var tokens = new List <TokenPair>();
void BeginElse(TokenMatch match)
{
var firstNonContentToken = tokens
.AsReadOnly()
.Reverse()
.FirstOrDefault(e => !e.Type.Equals(TokenType.Content));
if (!firstNonContentToken.Type.Equals(TokenType.IfClose))
{
context.Errors
.Add(new MorestachioSyntaxError(
context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, match.Value)), "find if block for else",
firstNonContentToken.Value, "{{/if}}", "Could not find an /if block for this else"));
}
else
{
scopestack.Push(new ScopeStackItem(TokenType.Else, firstNonContentToken.Value, match.Index));
tokens.Add(new TokenPair(TokenType.Else, firstNonContentToken.Value,
context.CurrentLocation));
}
}
void EndIf(TokenMatch match, string expected)
{
if (!scopestack.Any())
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, match.Value)),
"if",
"{{#if name}}"));
}
else
{
var item1 = scopestack.Peek();
if (item1.TokenType == TokenType.If || item1.TokenType == TokenType.IfNot)
{
var token = scopestack.Pop().Value;
tokens.Add(new TokenPair(TokenType.IfClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(
context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, match.Value)),
"if",
"{{#if name}}"));
}
}
}
string TrimToken(string token, string keyword, char key = '#')
{
token = token.TrimStart(key);
if (keyword != null)
{
token = token.Trim().Substring(keyword.Length);
}
return(token.Trim());
}
foreach (var match in MatchTokens(templateString, context))
{
var tokenValue = match.Value;
var trimmedToken = tokenValue
.Remove(0, context.PrefixToken.Length);
trimmedToken = trimmedToken.Remove(trimmedToken.Length - context.SuffixToken.Length);
if (context.CommentIntend > 0)
{
if (trimmedToken == "/!")
{
context.CommentIntend--;
if (context.CommentIntend == 0)
{
//move forward in the string.
if (context.Character > match.Index + match.Length)
{
throw new InvalidOperationException("Internal index location error");
}
context.SetLocation(match.Index + match.Length);
}
}
else if (trimmedToken.Equals("!"))
{
context.CommentIntend++;
}
}
else
{
//yield front content.
if (match.Index > context.Character)
{
tokens.Add(new TokenPair(TokenType.Content, templateString.Substring(context.Character, match.Index - context.Character),
context.CurrentLocation));
}
context.SetLocation(match.Index + context.PrefixToken.Length);
if (trimmedToken.StartsWith("#declare ", true, CultureInfo.InvariantCulture))
{
var token = TrimToken(trimmedToken, "declare ");
scopestack.Push(new ScopeStackItem(TokenType.PartialDeclarationOpen, token, match.Index));
if (string.IsNullOrWhiteSpace(token))
{
context.Errors.Add(new MorestachioSyntaxError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "open", "declare",
"{{#declare name}}", " Missing the Name."));
}
else
{
partialsNames.Add(token);
tokens.Add(new TokenPair(TokenType.PartialDeclarationOpen, token,
context.CurrentLocation));
}
}
else if (trimmedToken.Equals("/declare", StringComparison.CurrentCultureIgnoreCase))
{
if (scopestack.Any() && scopestack.Peek().TokenType == TokenType.PartialDeclarationOpen)
{
var token = scopestack.Pop().Value;
tokens.Add(new TokenPair(TokenType.PartialDeclarationClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "declare",
"{{#declare name}}"));
}
}
else if (trimmedToken.StartsWith("#include ", true, CultureInfo.InvariantCulture))
{
var token = trimmedToken.TrimStart('#').Trim();
var partialRegex = PartialIncludeRegEx.Match(token);
var partialName = partialRegex.Groups[1].Value;
var partialContext = partialRegex.Groups[2].Value;
if (!string.IsNullOrWhiteSpace(partialContext))
{
partialContext = token.Substring(partialRegex.Groups[2].Index + "WITH ".Length);
}
if (string.IsNullOrWhiteSpace(partialName) || !partialsNames.Contains(partialName))
{
context.Errors.Add(new MorestachioSyntaxError(
context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)),
"use",
"include",
"{{#include name}}",
$" There is no Partial declared '{partialName}'. Partial names are case sensitive and must be declared before an include."));
}
else
{
IMorestachioExpression exp = null;
if (!string.IsNullOrWhiteSpace(partialContext))
{
exp = ExpressionParser.ParseExpression(partialContext, context);
}
var tokenPair = new TokenPair(TokenType.RenderPartial, partialName, context.CurrentLocation, exp);
tokens.Add(tokenPair);
}
}
else if (trimmedToken.StartsWith("#each ", true, CultureInfo.InvariantCulture))
{
var token = TrimToken(trimmedToken, "each");
var eval = EvaluateNameFromToken(token);
token = eval.Value;
var alias = eval.Name;
scopestack.Push(new ScopeStackItem(TokenType.CollectionOpen, alias ?? token, match.Index));
if (token.Trim() != "")
{
token = token.Trim();
ScopingBehavior?scopeBehavior = null;
if (!string.IsNullOrWhiteSpace(alias))
{
if (token.EndsWith("NoScope", StringComparison.InvariantCultureIgnoreCase))
{
scopeBehavior = ScopingBehavior.DoNotScope;
}
if (token.EndsWith("WithScope", StringComparison.InvariantCultureIgnoreCase))
{
scopeBehavior = ScopingBehavior.ScopeAnyway;
}
}
tokens.Add(new TokenPair(TokenType.CollectionOpen,
token,
context.CurrentLocation,
ExpressionParser.ParseExpression(token, context), scopeBehavior));
}
else
{
context.Errors.Add(new InvalidPathSyntaxError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), ""));
}
if (!string.IsNullOrWhiteSpace(alias))
{
context.AdvanceLocation("each ".Length + alias.Length);
tokens.Add(new TokenPair(TokenType.Alias, alias,
context.CurrentLocation));
}
}
else if (trimmedToken.Equals("/each", StringComparison.InvariantCultureIgnoreCase))
{
if (scopestack.Any() && scopestack.Peek().TokenType == TokenType.CollectionOpen)
{
var token = scopestack.Pop().Value;
tokens.Add(new TokenPair(TokenType.CollectionClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "each", "{{#each name}}"));
}
}
else if (trimmedToken.StartsWith("#while ", true, CultureInfo.InvariantCulture))
{
var token = TrimToken(trimmedToken, "while");
scopestack.Push(new ScopeStackItem(TokenType.WhileLoopOpen, token, match.Index));
if (token.Trim() != "")
{
token = token.Trim();
tokens.Add(new TokenPair(TokenType.WhileLoopOpen,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
}
else
{
context.Errors.Add(new InvalidPathSyntaxError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), ""));
}
}
else if (trimmedToken.Equals("/while", StringComparison.InvariantCultureIgnoreCase))
{
if (scopestack.Any() && scopestack.Peek().TokenType == TokenType.WhileLoopOpen)
{
var token = scopestack.Pop().Value;
tokens.Add(new TokenPair(TokenType.WhileLoopClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "while", "{{#while Expression}}"));
}
}
else if (trimmedToken.StartsWith("#do ", true, CultureInfo.InvariantCulture))
{
var token = TrimToken(trimmedToken, "do");
scopestack.Push(new ScopeStackItem(TokenType.DoLoopOpen, token, match.Index));
if (token.Trim() != "")
{
token = token.Trim();
tokens.Add(new TokenPair(TokenType.DoLoopOpen,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
}
else
{
context.Errors.Add(new InvalidPathSyntaxError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), ""));
}
}
else if (trimmedToken.Equals("/do", StringComparison.InvariantCultureIgnoreCase))
{
if (scopestack.Any() && scopestack.Peek().TokenType == TokenType.DoLoopOpen)
{
var token = scopestack.Pop().Value;
tokens.Add(new TokenPair(TokenType.DoLoopClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "do", "{{#do Expression}}"));
}
}
else if (trimmedToken.StartsWith("#repeat ", true, CultureInfo.InvariantCulture))
{
var token = TrimToken(trimmedToken, "repeat");
scopestack.Push(new ScopeStackItem(TokenType.RepeatLoopOpen, token, match.Index));
if (token.Trim() != "")
{
token = token.Trim();
tokens.Add(new TokenPair(TokenType.RepeatLoopOpen,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
}
else
{
context.Errors.Add(new InvalidPathSyntaxError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), ""));
}
}
else if (trimmedToken.Equals("/repeat", StringComparison.InvariantCultureIgnoreCase))
{
if (scopestack.Any() && scopestack.Peek().TokenType == TokenType.RepeatLoopOpen)
{
var token = scopestack.Pop().Value;
tokens.Add(new TokenPair(TokenType.RepeatLoopClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "repeat", "{{#repeat Expression}}"));
}
}
else if (trimmedToken.StartsWith("#if ", true, CultureInfo.InvariantCulture))
{
var token = TrimToken(trimmedToken, "if");
var eval = EvaluateNameFromToken(token);
token = eval.Value;
if (eval.Name != null)
{
context.Errors.Add(new MorestachioSyntaxError(
context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "^if", "AS", "No Alias"));
}
scopestack.Push(new ScopeStackItem(TokenType.If, token, match.Index));
if (token.Trim() != "")
{
token = token.Trim();
tokens.Add(new TokenPair(TokenType.If,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
}
else
{
context.Errors.Add(new InvalidPathSyntaxError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), ""));
}
}
else if (trimmedToken.StartsWith("^if ", true, CultureInfo.InvariantCulture))
{
var token = TrimToken(trimmedToken, "if", '^');
var eval = EvaluateNameFromToken(token);
token = eval.Value;
if (eval.Name != null)
{
context.Errors.Add(new MorestachioSyntaxError(
context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "^if", "AS", "No Alias"));
}
scopestack.Push(new ScopeStackItem(TokenType.IfNot, token, match.Index));
if (token.Trim() != "")
{
token = token.Trim();
tokens.Add(new TokenPair(TokenType.IfNot,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
}
else
{
context.Errors.Add(new InvalidPathSyntaxError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), ""));
}
}
else if (trimmedToken.Equals("/if", StringComparison.InvariantCultureIgnoreCase))
{
EndIf(match, "/If");
}
else if (trimmedToken.Equals("#ifelse", StringComparison.InvariantCultureIgnoreCase))
{
EndIf(match, "#ifelse");
BeginElse(match);
}
else if (trimmedToken.Equals("#else", StringComparison.InvariantCultureIgnoreCase))
{
BeginElse(match);
}
else if (trimmedToken.Equals("/else", StringComparison.InvariantCultureIgnoreCase))
{
if (scopestack.Any() && scopestack.Peek().TokenType == TokenType.Else)
{
var token = scopestack.Pop().Value;
tokens.Add(new TokenPair(TokenType.ElseClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(
context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "else",
"{{#else name}}"));
}
}
else if (trimmedToken.StartsWith("#var ", true, CultureInfo.InvariantCulture))
{
tokens.Add(ExpressionParser.TokenizeVariableAssignment(trimmedToken,
context, TokenType.VariableVar));
}
else if (trimmedToken.StartsWith("#let ", true, CultureInfo.InvariantCulture))
{
tokens.Add(ExpressionParser.TokenizeVariableAssignment(trimmedToken,
context, TokenType.VariableLet));
}
else if (trimmedToken.StartsWith("^"))
{
//open inverted group
var token = trimmedToken.TrimStart('^').Trim();
var eval = EvaluateNameFromToken(token);
token = eval.Value;
var alias = eval.Name;
scopestack.Push(new ScopeStackItem(TokenType.InvertedElementOpen, alias ?? token, match.Index));
tokens.Add(new TokenPair(TokenType.InvertedElementOpen,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
if (!string.IsNullOrWhiteSpace(alias))
{
context.AdvanceLocation(1 + alias.Length);
tokens.Add(new TokenPair(TokenType.Alias, alias,
context.CurrentLocation));
}
}
else if (trimmedToken.StartsWith("&"))
{
//escaped single element
var token = trimmedToken.TrimStart('&').Trim();
tokens.Add(new TokenPair(TokenType.UnescapedSingleValue,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
}
else if (trimmedToken.StartsWith("!"))
{
//it's a comment drop this on the floor, no need to even yield it.
if (trimmedToken.Equals("!"))
{
//except for when its a block comment then set the isCommentBlock flag
context.CommentIntend++;
}
}
else if (trimmedToken.Equals("#NL", StringComparison.InvariantCultureIgnoreCase))
{
tokens.Add(new TokenPair(TokenType.WriteLineBreak, trimmedToken, context.CurrentLocation));
}
else if (trimmedToken.Equals("#TNL", StringComparison.InvariantCultureIgnoreCase))
{
tokens.Add(new TokenPair(TokenType.TrimLineBreak, trimmedToken, context.CurrentLocation));
}
else if (trimmedToken.Equals("#TNLS", StringComparison.InvariantCultureIgnoreCase))
{
tokens.Add(new TokenPair(TokenType.TrimLineBreaks, trimmedToken, context.CurrentLocation));
}
else if (trimmedToken.Equals("#TRIMALL", StringComparison.InvariantCultureIgnoreCase))
{
tokens.Add(new TokenPair(TokenType.TrimEverything, trimmedToken, context.CurrentLocation));
}
else if (trimmedToken.StartsWith("#SET OPTION ", StringComparison.InvariantCultureIgnoreCase))
{
var token = TrimToken(trimmedToken, "SET OPTION ");
var expectEquals = false;
string name = null;
IMorestachioExpression value = null;
for (int i = 0; i < token.Length; i++)
{
var c = token[i];
if (IsWhiteSpaceDelimiter(c))
{
expectEquals = true;
continue;
}
if (expectEquals || c == '=')
{
if (c != '=')
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "/", "{{#SET OPTION Name = Value}}",
$" Expected to find '=' or whitespace after name but found '{c}'"));
}
else
{
name = token.Substring(0, i - 1).Trim();
value = ExpressionParser.ParseExpression(token.Substring(i + 1).Trim(), context);
break;
}
}
}
if (string.IsNullOrWhiteSpace(name))
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "/", "{{#SET OPTION Name = Value}}",
$" Expected to find '=' after name"));
break;
}
if (value == null)
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "/", "{{#SET OPTION Name = Value}}",
$" Expected to find an expression after '='"));
break;
}
await context.SetOption(name, value, parserOptions);
}
//else if (tokenValue.Equals("{{/TRIMALL}}", StringComparison.InvariantCultureIgnoreCase))
//{
// tokens.Add(new TokenPair(TokenType.StopTrimEverything, tokenValue, context.CurrentLocation));
//}
else
{
//check for custom DocumentItem provider
var customDocumentProvider =
parserOptions.CustomDocumentItemProviders.FirstOrDefault(e => e.ShouldTokenize(trimmedToken));
if (customDocumentProvider != null)
{
var tokenPairs = customDocumentProvider
.Tokenize(new CustomDocumentItemProvider.TokenInfo(trimmedToken, context, scopestack),
parserOptions);
tokens.AddRange(tokenPairs);
}
else if (trimmedToken.StartsWith("#"))
{
//open group
var token = trimmedToken.TrimStart('#').Trim();
var eval = EvaluateNameFromToken(token);
token = eval.Value;
var alias = eval.Name;
scopestack.Push(new ScopeStackItem(TokenType.ElementOpen, alias ?? token, match.Index));
tokens.Add(new TokenPair(TokenType.ElementOpen,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
if (!string.IsNullOrWhiteSpace(alias))
{
context.AdvanceLocation(3 + alias.Length);
tokens.Add(new TokenPair(TokenType.Alias, alias,
context.CurrentLocation));
}
}
else if (trimmedToken.StartsWith("/"))
{
var token = trimmedToken.TrimStart('/').Trim();
//close group
if (!scopestack.Any())
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "/", "{{#path}}",
" There are more closing elements then open."));
}
else
{
var item = scopestack.Peek();
if ((item.TokenType == TokenType.ElementOpen ||
item.TokenType == TokenType.InvertedElementOpen) &&
item.Value == token)
{
scopestack.Pop();
tokens.Add(new TokenPair(TokenType.ElementClose, token,
context.CurrentLocation));
}
else
{
context.Errors.Add(new MorestachioUnopendScopeError(context.CurrentLocation
.AddWindow(new CharacterSnippedLocation(1, 1, tokenValue)), "/", "{{#path}}",
" There are more closing elements then open."));
}
}
}
else
{
//unsingle value.
var token = trimmedToken.Trim();
tokens.Add(new TokenPair(TokenType.EscapedSingleValue,
token,
context.CurrentLocation, ExpressionParser.ParseExpression(token, context)));
}
}
//move forward in the string.
if (context.Character > match.Index + match.Length)
{
throw new InvalidOperationException("Internal index location error");
}
context.SetLocation(match.Index + match.Length);
}
}
if (context.Character < templateString.Length)
{
tokens.Add(new TokenPair(TokenType.Content, templateString.Substring(context.Character),
context.CurrentLocation));
}
if (scopestack.Any() || parserOptions.CustomDocumentItemProviders.Any(f => f.ScopeStack.Any()))
{
foreach (var unclosedScope in scopestack
.Concat(parserOptions.CustomDocumentItemProviders.SelectMany(f => f.ScopeStack))
.Select(k =>
{
return(new
{
scope = k.TokenType.ToString(),
location = HumanizeCharacterLocation(k.Index, context.Lines)
});
}).Reverse())
{
context.Errors.Add(new MorestachioUnclosedScopeError(unclosedScope.location
.AddWindow(new CharacterSnippedLocation(1, -1, "")), unclosedScope.scope, ""));
}
}
return(new TokenizerResult(tokens));
}
public TokenPairDebuggerProxy(TokenPair pair)
{
_pair = pair;
}
