private string GetRepeatingMultilinePreamble(PreToken token)
{
if (token.Repeating == false)
{
return(null);
}
if (string.IsNullOrEmpty(token.Preamble))
{
return(null);
}
if (token.Preamble.IndexOf('\n') == -1)
{
return(null);
}
var pre = token.Preamble.SubstringBeforeLastString("\n");
var post = token.Preamble.SubstringAfterLastString("\n");
if (string.IsNullOrWhiteSpace(pre) == false &&
string.IsNullOrWhiteSpace(post))
{
return("\n" + post);
}
return(null);
}
private void ParseTokenValueInDoubleQuotes(PreTokenEnumerator enumerator, ref PreToken token, ref FlatTokenParserState state, ref StringBuilder tokenContent)
{
var next = enumerator.Next();
switch (next)
{
case @"""":
state = FlatTokenParserState.InTokenValueRunOff;
break;
default:
token.AppendValue(next);
break;
}
tokenContent.Append(next);
}
private void AppendToken(PreTemplate template, PreToken token, ref StringBuilder tokenContent, TokenizerOptions options)
{
token.Content = tokenContent.ToString();
token.Id = template.Tokens.Count + 1;
token.IsNull = string.Compare(token.Name, "null", StringComparison.InvariantCultureIgnoreCase) == 0;
if (options.TrimPreambleBeforeNewLine)
{
token.TrimPreambleBeforeNewLine();
}
if (options.TerminateOnNewline)
{
token.TerminateOnNewline = true;
}
tokenContent.Clear();
var preamble = GetRepeatingMultilinePreamble(token);
if (string.IsNullOrEmpty(preamble) == false && token.Repeating)
{
token.Repeating = false;
template.Tokens.Add(token);
var repeat = new PreToken
{
Optional = true,
Repeating = true,
TerminateOnNewline = token.TerminateOnNewline,
Content = token.Content
};
repeat.AppendName(token.Name);
repeat.AppendPreamble(preamble);
repeat.AppendDecorators(token.Decorators);
repeat.Id = template.Tokens.Count + 1;
repeat.DependsOnId = token.Id;
template.Tokens.Add(repeat);
}
else
{
template.Tokens.Add(token);
}
}
private void ParseTokenValueRunOff(PreTokenEnumerator enumerator, ref PreTemplate template, ref PreToken token, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options)
{
var next = enumerator.Next();
tokenContent.Append(next);
if (string.IsNullOrWhiteSpace(next))
{
if (inFrontMatterToken == false)
{
return;
}
if (next != "\n")
{
return;
}
}
switch (next)
{
case ":":
state = FlatTokenParserState.InDecorator;
break;
case "}" when inFrontMatterToken == false:
case "\n" when inFrontMatterToken:
token.IsFrontMatterToken = inFrontMatterToken;
AppendToken(template, token, ref tokenContent, options);
token = new PreToken();
if (inFrontMatterToken)
{
inFrontMatterToken = false;
state = FlatTokenParserState.InFrontMatter;
}
else
{
state = FlatTokenParserState.InPreamble;
}
break;
default:
throw new TokenizerException($"Unexpected character: '{next}'");
}
}
private void ParsePreamble(ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref StringBuilder tokenContent)
{
var next = enumerator.Next();
switch (next)
{
case "{":
if (enumerator.Peek() == "{")
{
token.AppendPreamble("{");
enumerator.Next();
}
else
{
token.Location = enumerator.Location.Clone();
tokenContent.Append("{");
state = FlatTokenParserState.InTokenName;
}
break;
case "}":
if (enumerator.Peek() == "}")
{
token.AppendPreamble("}");
enumerator.Next();
break;
}
throw new ParsingException($"Unescaped character '}}' in template.", enumerator);
default:
token.AppendPreamble(next);
break;
}
}
private void ParseTokenDecorators(PreToken preToken, Token token)
{
// If pre-token has value set, add transformer to set it when parsing
if (string.IsNullOrEmpty(preToken.Value) == false)
{
var setContext = new TokenDecoratorContext(typeof(SetTransformer));
setContext.Parameters.Add(preToken.Value);
token.Decorators.Add(setContext);
}
foreach (var decorator in preToken.Decorators)
{
if (IsConcatenationDecorator(preToken.Name, decorator, out var joiningString))
{
token.Concatenate = true;
token.ConcatenationString = joiningString;
continue;
}
TokenDecoratorContext context = null;
foreach (var operatorType in transformers)
{
if (string.Compare(decorator.Name, operatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0 ||
string.Compare($"{decorator.Name}Transformer", operatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0)
{
if (decorator.IsNotDecorator)
{
throw new TokenizerException($"{decorator.Name} cannot be prefixed with '!' character.");
}
context = new TokenDecoratorContext(operatorType);
foreach (var arg in decorator.Args)
{
context.Parameters.Add(arg);
}
token.Decorators.Add(context);
break;
}
}
if (context != null)
{
continue;
}
foreach (var validatorType in validators)
{
if (string.Compare(decorator.Name, validatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0 ||
string.Compare($"{decorator.Name}Validator", validatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0)
{
context = new TokenDecoratorContext(validatorType);
foreach (var arg in decorator.Args)
{
context.Parameters.Add(arg);
}
context.IsNotValidator = decorator.IsNotDecorator;
token.Decorators.Add(context);
break;
}
}
if (context == null)
{
throw new TokenizerException($"Unknown Token Operation: {decorator.Name}");
}
}
if (preToken.IsFrontMatterToken)
{
var hasSetTransformer = token.Decorators.Any(d => d.DecoratorType == typeof(SetTransformer));
if (hasSetTransformer == false)
{
throw new TokenizerException($"Front Matter Token '{preToken.Name}' must have an assignment operation.");
}
}
}
private void AppendDecorator(PreTokenEnumerator enumerator, PreToken token, PreTokenDecorator decorator)
{
if (decorator == null)
{
return;
}
if (string.IsNullOrEmpty(decorator.Name))
{
return;
}
switch (decorator.Name.ToLowerInvariant())
{
case "eol":
case "$":
if (decorator.Args.Any())
{
throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator);
}
token.TerminateOnNewline = true;
break;
case "optional":
case "?":
if (decorator.Args.Any())
{
throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator);
}
token.Optional = true;
break;
case "repeating":
case "*":
if (decorator.Args.Any())
{
throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator);
}
token.Repeating = true;
break;
case "required":
case "!":
if (decorator.Args.Any())
{
throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator);
}
token.Required = true;
break;
case "once":
if (decorator.Args.Any())
{
throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator);
}
token.ConsiderOnce = true;
break;
default:
token.Decorators.Add(decorator);
break;
}
}
private void ParseDecorator(PreTemplate template, ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref PreTokenDecorator decorator, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options)
{
var next = enumerator.Next();
tokenContent.Append(next);
if (string.IsNullOrWhiteSpace(next))
{
if (inFrontMatterToken == false)
{
return;
}
if (next != "\n")
{
return;
}
}
switch (next)
{
case "}" when inFrontMatterToken == false:
case "\n" when inFrontMatterToken:
token.IsFrontMatterToken = inFrontMatterToken;
AppendDecorator(enumerator, token, decorator);
AppendToken(template, token, ref tokenContent, options);
token = new PreToken();
decorator = new PreTokenDecorator();
if (inFrontMatterToken)
{
inFrontMatterToken = false;
state = FlatTokenParserState.InFrontMatter;
}
else
{
state = FlatTokenParserState.InPreamble;
}
break;
case ",":
AppendDecorator(enumerator, token, decorator);
decorator = new PreTokenDecorator();
break;
case "(":
state = FlatTokenParserState.InDecoratorArgument;
break;
case "}" when inFrontMatterToken:
case "\n" when inFrontMatterToken == false:
throw new ParsingException($"'{decorator.Name}' unexpected character: {next}", enumerator);
case "!":
if (string.IsNullOrWhiteSpace(decorator.Name))
{
decorator.IsNotDecorator = true;
}
else
{
throw new ParsingException($"'{decorator.Name}' unexpected character: {next}", enumerator);
}
break;
default:
decorator.AppendName(next);
break;
}
}
private void ParseTokenValue(PreTemplate template, ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options)
{
var next = enumerator.Next();
var peek = enumerator.Peek();
tokenContent.Append(next);
switch (next)
{
case "{":
throw new ParsingException($"Unexpected character '{{' in token '{token.Name}'", enumerator);
case "}" when inFrontMatterToken == false:
case "\n" when inFrontMatterToken:
token.IsFrontMatterToken = inFrontMatterToken;
AppendToken(template, token, ref tokenContent, options);
token = new PreToken();
if (inFrontMatterToken)
{
inFrontMatterToken = false;
state = FlatTokenParserState.InFrontMatter;
}
else
{
state = FlatTokenParserState.InPreamble;
}
break;
case ":":
state = FlatTokenParserState.InDecorator;
break;
case "'":
state = FlatTokenParserState.InTokenValueSingleQuotes;
break;
case "\"":
state = FlatTokenParserState.InTokenValueDoubleQuotes;
break;
case " ":
switch (peek)
{
case " ":
case "}" when inFrontMatterToken == false:
case "\n" when inFrontMatterToken:
case ":":
break;
default:
if (token.HasValue)
{
throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator);
}
break;
}
break;
case "}" when inFrontMatterToken:
case "\n" when inFrontMatterToken == false:
throw new ParsingException($"'{token.Name}' unexpected character: {next}", enumerator);
default:
token.AppendValue(next);
break;
}
}
private void ParseTokenName(PreTemplate template, ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options)
{
var next = enumerator.Next();
var peek = enumerator.Peek();
tokenContent.Append(next);
switch (next)
{
case "{":
throw new ParsingException($"Unexpected character '{{' in token '{token.Name}'", enumerator);
case "}":
if (inFrontMatterToken)
{
throw new ParsingException($"Invalid character '{next}' in token '{token.Name}'", enumerator);
}
else
{
AppendToken(template, token, ref tokenContent, options);
token = new PreToken();
state = FlatTokenParserState.InPreamble;
}
break;
case "$":
token.TerminateOnNewline = true;
switch (peek)
{
case " ":
case "?":
case "*":
case "}":
case ":":
case "!":
break;
default:
throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator);
}
break;
case "?":
token.Optional = true;
switch (peek)
{
case " ":
case "$":
case "*":
case "}":
case ":":
case "!":
break;
default:
throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator);
}
if (token.Required)
{
throw new ParsingException($"Required token {token.Name} can't be Optional", enumerator);
}
break;
case "*":
token.Repeating = true;
token.Optional = true;
switch (peek)
{
case " ":
case "$":
case "?":
case "}":
case ":":
case "!":
break;
default:
throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator);
}
break;
case "!":
token.Required = true;
switch (peek)
{
case " ":
case "*":
case "$":
case "?":
case "}":
case ":":
break;
default:
throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator);
}
if (token.Optional)
{
throw new ParsingException($"Optional token {token.Name} can't be Required", enumerator);
}
break;
case ":":
state = FlatTokenParserState.InDecorator;
break;
case "=":
state = FlatTokenParserState.InTokenValue;
break;
case " ":
switch (peek)
{
case " ":
case "*":
case "$":
case "?":
case "}":
case ":":
case "!":
case "=":
break;
case "\n" when inFrontMatterToken:
break;
default:
if (string.IsNullOrWhiteSpace(token.Name) == false)
{
throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator);
}
break;
}
break;
case "\n":
if (inFrontMatterToken)
{
token.IsFrontMatterToken = true;
AppendToken(template, token, ref tokenContent, options);
token = new PreToken();
inFrontMatterToken = false;
state = FlatTokenParserState.InFrontMatter;
}
else
{
throw new ParsingException($"Invalid character '{next}' in token '{token.Name}'", enumerator);
}
break;
default:
if (ValidTokenNameCharacters.Contains(next))
{
token.AppendName(next);
}
else
{
throw new ParsingException($"Invalid character '{next}' in token '{token.Name}'", enumerator);
}
break;
}
}
/// <summary>
/// Parses the template string and constructs a <see cref="PreTemplate"/>.
/// </summary>
public PreTemplate Parse(string template, TokenizerOptions options)
{
var preTemplate = new PreTemplate {
Options = options.Clone()
};
var enumerator = new PreTokenEnumerator(template);
if (enumerator.IsEmpty)
{
return(preTemplate);
}
var state = FlatTokenParserState.AtStart;
var token = new PreToken();
var decorator = new PreTokenDecorator();
var argument = string.Empty;
var tokenContent = new StringBuilder();
var frontMatterName = new StringBuilder();
var frontMatterValue = new StringBuilder();
var inFrontMatterToken = false;
// Basic State Machine to parse the template input
while (enumerator.IsEmpty == false)
{
switch (state)
{
case FlatTokenParserState.AtStart:
ParseStart(enumerator, ref state);
break;
case FlatTokenParserState.InFrontMatter:
ParseFrontMatter(enumerator, ref frontMatterName, ref state);
break;
case FlatTokenParserState.InFrontMatterComment:
ParseFrontMatterComment(enumerator, ref state);
break;
case FlatTokenParserState.InFrontMatterOption:
ParseFrontMatterOption(enumerator, ref frontMatterName, ref state, ref inFrontMatterToken, ref token);
break;
case FlatTokenParserState.InFrontMatterOptionValue:
ParseFrontMatterOptionValue(preTemplate, enumerator, ref frontMatterName, ref frontMatterValue, ref state);
break;
case FlatTokenParserState.InPreamble:
ParsePreamble(ref token, enumerator, ref state, ref tokenContent);
break;
case FlatTokenParserState.InTokenName:
ParseTokenName(preTemplate, ref token, enumerator, ref state, ref inFrontMatterToken, ref tokenContent, preTemplate.Options);
break;
case FlatTokenParserState.InTokenValue:
ParseTokenValue(preTemplate, ref token, enumerator, ref state, ref inFrontMatterToken, ref tokenContent, preTemplate.Options);
break;
case FlatTokenParserState.InTokenValueSingleQuotes:
ParseTokenValueInSingleQuotes(enumerator, ref token, ref state, ref tokenContent);
break;
case FlatTokenParserState.InTokenValueDoubleQuotes:
ParseTokenValueInDoubleQuotes(enumerator, ref token, ref state, ref tokenContent);
break;
case FlatTokenParserState.InTokenValueRunOff:
ParseTokenValueRunOff(enumerator, ref preTemplate, ref token, ref state, ref inFrontMatterToken, ref tokenContent, preTemplate.Options);
break;
case FlatTokenParserState.InDecorator:
ParseDecorator(preTemplate, ref token, enumerator, ref state, ref decorator, ref inFrontMatterToken, ref tokenContent, preTemplate.Options);
break;
case FlatTokenParserState.InDecoratorArgument:
ParseDecoratorArgument(enumerator, ref state, ref decorator, ref argument, ref tokenContent);
break;
case FlatTokenParserState.InDecoratorArgumentSingleQuotes:
ParseDecoratorArgumentInSingleQuotes(enumerator, ref state, ref decorator, ref argument, ref tokenContent);
break;
case FlatTokenParserState.InDecoratorArgumentDoubleQuotes:
ParseDecoratorArgumentInDoubleQuotes(enumerator, ref state, ref decorator, ref argument, ref tokenContent);
break;
case FlatTokenParserState.InDecoratorArgumentRunOff:
ParseDecoratorArgumentRunOff(enumerator, ref state, ref tokenContent);
break;
default:
throw new TokenizerException($"Unknown FlatTokenParserState: {state}");
}
}
// Append current token if it has contents
// Note: allow empty token values, as these will serve to truncate the last
// token in the template
if (string.IsNullOrWhiteSpace(token.Preamble) == false)
{
AppendToken(preTemplate, token, ref tokenContent, preTemplate.Options);
}
return(preTemplate);
}
private void ParseFrontMatterOption(PreTokenEnumerator enumerator, ref StringBuilder frontMatterName, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref PreToken token)
{
var next = enumerator.Next();
switch (next)
{
case ":":
if (frontMatterName.ToString().Trim().ToLowerInvariant() == "set")
{
inFrontMatterToken = true;
frontMatterName.Clear();
token.Location = enumerator.Location.Clone();
state = FlatTokenParserState.InTokenName;
}
else
{
state = FlatTokenParserState.InFrontMatterOptionValue;
}
break;
default:
frontMatterName.Append(next);
break;
}
}
