private string GetRepeatingMultilinePreamble(PreToken token) { if (token.Repeating == false) { return(null); } if (string.IsNullOrEmpty(token.Preamble)) { return(null); } if (token.Preamble.IndexOf('\n') == -1) { return(null); } var pre = token.Preamble.SubstringBeforeLastString("\n"); var post = token.Preamble.SubstringAfterLastString("\n"); if (string.IsNullOrWhiteSpace(pre) == false && string.IsNullOrWhiteSpace(post)) { return("\n" + post); } return(null); }
private void ParseTokenValueInDoubleQuotes(PreTokenEnumerator enumerator, ref PreToken token, ref FlatTokenParserState state, ref StringBuilder tokenContent) { var next = enumerator.Next(); switch (next) { case @"""": state = FlatTokenParserState.InTokenValueRunOff; break; default: token.AppendValue(next); break; } tokenContent.Append(next); }
private void AppendToken(PreTemplate template, PreToken token, ref StringBuilder tokenContent, TokenizerOptions options) { token.Content = tokenContent.ToString(); token.Id = template.Tokens.Count + 1; token.IsNull = string.Compare(token.Name, "null", StringComparison.InvariantCultureIgnoreCase) == 0; if (options.TrimPreambleBeforeNewLine) { token.TrimPreambleBeforeNewLine(); } if (options.TerminateOnNewline) { token.TerminateOnNewline = true; } tokenContent.Clear(); var preamble = GetRepeatingMultilinePreamble(token); if (string.IsNullOrEmpty(preamble) == false && token.Repeating) { token.Repeating = false; template.Tokens.Add(token); var repeat = new PreToken { Optional = true, Repeating = true, TerminateOnNewline = token.TerminateOnNewline, Content = token.Content }; repeat.AppendName(token.Name); repeat.AppendPreamble(preamble); repeat.AppendDecorators(token.Decorators); repeat.Id = template.Tokens.Count + 1; repeat.DependsOnId = token.Id; template.Tokens.Add(repeat); } else { template.Tokens.Add(token); } }
private void ParseTokenValueRunOff(PreTokenEnumerator enumerator, ref PreTemplate template, ref PreToken token, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options) { var next = enumerator.Next(); tokenContent.Append(next); if (string.IsNullOrWhiteSpace(next)) { if (inFrontMatterToken == false) { return; } if (next != "\n") { return; } } switch (next) { case ":": state = FlatTokenParserState.InDecorator; break; case "}" when inFrontMatterToken == false: case "\n" when inFrontMatterToken: token.IsFrontMatterToken = inFrontMatterToken; AppendToken(template, token, ref tokenContent, options); token = new PreToken(); if (inFrontMatterToken) { inFrontMatterToken = false; state = FlatTokenParserState.InFrontMatter; } else { state = FlatTokenParserState.InPreamble; } break; default: throw new TokenizerException($"Unexpected character: '{next}'"); } }
private void ParsePreamble(ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref StringBuilder tokenContent) { var next = enumerator.Next(); switch (next) { case "{": if (enumerator.Peek() == "{") { token.AppendPreamble("{"); enumerator.Next(); } else { token.Location = enumerator.Location.Clone(); tokenContent.Append("{"); state = FlatTokenParserState.InTokenName; } break; case "}": if (enumerator.Peek() == "}") { token.AppendPreamble("}"); enumerator.Next(); break; } throw new ParsingException($"Unescaped character '}}' in template.", enumerator); default: token.AppendPreamble(next); break; } }
private void ParseTokenDecorators(PreToken preToken, Token token) { // If pre-token has value set, add transformer to set it when parsing if (string.IsNullOrEmpty(preToken.Value) == false) { var setContext = new TokenDecoratorContext(typeof(SetTransformer)); setContext.Parameters.Add(preToken.Value); token.Decorators.Add(setContext); } foreach (var decorator in preToken.Decorators) { if (IsConcatenationDecorator(preToken.Name, decorator, out var joiningString)) { token.Concatenate = true; token.ConcatenationString = joiningString; continue; } TokenDecoratorContext context = null; foreach (var operatorType in transformers) { if (string.Compare(decorator.Name, operatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0 || string.Compare($"{decorator.Name}Transformer", operatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0) { if (decorator.IsNotDecorator) { throw new TokenizerException($"{decorator.Name} cannot be prefixed with '!' character."); } context = new TokenDecoratorContext(operatorType); foreach (var arg in decorator.Args) { context.Parameters.Add(arg); } token.Decorators.Add(context); break; } } if (context != null) { continue; } foreach (var validatorType in validators) { if (string.Compare(decorator.Name, validatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0 || string.Compare($"{decorator.Name}Validator", validatorType.Name, StringComparison.InvariantCultureIgnoreCase) == 0) { context = new TokenDecoratorContext(validatorType); foreach (var arg in decorator.Args) { context.Parameters.Add(arg); } context.IsNotValidator = decorator.IsNotDecorator; token.Decorators.Add(context); break; } } if (context == null) { throw new TokenizerException($"Unknown Token Operation: {decorator.Name}"); } } if (preToken.IsFrontMatterToken) { var hasSetTransformer = token.Decorators.Any(d => d.DecoratorType == typeof(SetTransformer)); if (hasSetTransformer == false) { throw new TokenizerException($"Front Matter Token '{preToken.Name}' must have an assignment operation."); } } }
private void AppendDecorator(PreTokenEnumerator enumerator, PreToken token, PreTokenDecorator decorator) { if (decorator == null) { return; } if (string.IsNullOrEmpty(decorator.Name)) { return; } switch (decorator.Name.ToLowerInvariant()) { case "eol": case "$": if (decorator.Args.Any()) { throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator); } token.TerminateOnNewline = true; break; case "optional": case "?": if (decorator.Args.Any()) { throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator); } token.Optional = true; break; case "repeating": case "*": if (decorator.Args.Any()) { throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator); } token.Repeating = true; break; case "required": case "!": if (decorator.Args.Any()) { throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator); } token.Required = true; break; case "once": if (decorator.Args.Any()) { throw new ParsingException($"'{decorator.Name}' decorator does not take any arguments", enumerator); } token.ConsiderOnce = true; break; default: token.Decorators.Add(decorator); break; } }
private void ParseDecorator(PreTemplate template, ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref PreTokenDecorator decorator, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options) { var next = enumerator.Next(); tokenContent.Append(next); if (string.IsNullOrWhiteSpace(next)) { if (inFrontMatterToken == false) { return; } if (next != "\n") { return; } } switch (next) { case "}" when inFrontMatterToken == false: case "\n" when inFrontMatterToken: token.IsFrontMatterToken = inFrontMatterToken; AppendDecorator(enumerator, token, decorator); AppendToken(template, token, ref tokenContent, options); token = new PreToken(); decorator = new PreTokenDecorator(); if (inFrontMatterToken) { inFrontMatterToken = false; state = FlatTokenParserState.InFrontMatter; } else { state = FlatTokenParserState.InPreamble; } break; case ",": AppendDecorator(enumerator, token, decorator); decorator = new PreTokenDecorator(); break; case "(": state = FlatTokenParserState.InDecoratorArgument; break; case "}" when inFrontMatterToken: case "\n" when inFrontMatterToken == false: throw new ParsingException($"'{decorator.Name}' unexpected character: {next}", enumerator); case "!": if (string.IsNullOrWhiteSpace(decorator.Name)) { decorator.IsNotDecorator = true; } else { throw new ParsingException($"'{decorator.Name}' unexpected character: {next}", enumerator); } break; default: decorator.AppendName(next); break; } }
private void ParseTokenValue(PreTemplate template, ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options) { var next = enumerator.Next(); var peek = enumerator.Peek(); tokenContent.Append(next); switch (next) { case "{": throw new ParsingException($"Unexpected character '{{' in token '{token.Name}'", enumerator); case "}" when inFrontMatterToken == false: case "\n" when inFrontMatterToken: token.IsFrontMatterToken = inFrontMatterToken; AppendToken(template, token, ref tokenContent, options); token = new PreToken(); if (inFrontMatterToken) { inFrontMatterToken = false; state = FlatTokenParserState.InFrontMatter; } else { state = FlatTokenParserState.InPreamble; } break; case ":": state = FlatTokenParserState.InDecorator; break; case "'": state = FlatTokenParserState.InTokenValueSingleQuotes; break; case "\"": state = FlatTokenParserState.InTokenValueDoubleQuotes; break; case " ": switch (peek) { case " ": case "}" when inFrontMatterToken == false: case "\n" when inFrontMatterToken: case ":": break; default: if (token.HasValue) { throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator); } break; } break; case "}" when inFrontMatterToken: case "\n" when inFrontMatterToken == false: throw new ParsingException($"'{token.Name}' unexpected character: {next}", enumerator); default: token.AppendValue(next); break; } }
private void ParseTokenName(PreTemplate template, ref PreToken token, PreTokenEnumerator enumerator, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref StringBuilder tokenContent, TokenizerOptions options) { var next = enumerator.Next(); var peek = enumerator.Peek(); tokenContent.Append(next); switch (next) { case "{": throw new ParsingException($"Unexpected character '{{' in token '{token.Name}'", enumerator); case "}": if (inFrontMatterToken) { throw new ParsingException($"Invalid character '{next}' in token '{token.Name}'", enumerator); } else { AppendToken(template, token, ref tokenContent, options); token = new PreToken(); state = FlatTokenParserState.InPreamble; } break; case "$": token.TerminateOnNewline = true; switch (peek) { case " ": case "?": case "*": case "}": case ":": case "!": break; default: throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator); } break; case "?": token.Optional = true; switch (peek) { case " ": case "$": case "*": case "}": case ":": case "!": break; default: throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator); } if (token.Required) { throw new ParsingException($"Required token {token.Name} can't be Optional", enumerator); } break; case "*": token.Repeating = true; token.Optional = true; switch (peek) { case " ": case "$": case "?": case "}": case ":": case "!": break; default: throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator); } break; case "!": token.Required = true; switch (peek) { case " ": case "*": case "$": case "?": case "}": case ":": break; default: throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator); } if (token.Optional) { throw new ParsingException($"Optional token {token.Name} can't be Required", enumerator); } break; case ":": state = FlatTokenParserState.InDecorator; break; case "=": state = FlatTokenParserState.InTokenValue; break; case " ": switch (peek) { case " ": case "*": case "$": case "?": case "}": case ":": case "!": case "=": break; case "\n" when inFrontMatterToken: break; default: if (string.IsNullOrWhiteSpace(token.Name) == false) { throw new ParsingException($"Invalid character '{peek}' in token '{token.Name}'", enumerator); } break; } break; case "\n": if (inFrontMatterToken) { token.IsFrontMatterToken = true; AppendToken(template, token, ref tokenContent, options); token = new PreToken(); inFrontMatterToken = false; state = FlatTokenParserState.InFrontMatter; } else { throw new ParsingException($"Invalid character '{next}' in token '{token.Name}'", enumerator); } break; default: if (ValidTokenNameCharacters.Contains(next)) { token.AppendName(next); } else { throw new ParsingException($"Invalid character '{next}' in token '{token.Name}'", enumerator); } break; } }
/// <summary> /// Parses the template string and constructs a <see cref="PreTemplate"/>. /// </summary> public PreTemplate Parse(string template, TokenizerOptions options) { var preTemplate = new PreTemplate { Options = options.Clone() }; var enumerator = new PreTokenEnumerator(template); if (enumerator.IsEmpty) { return(preTemplate); } var state = FlatTokenParserState.AtStart; var token = new PreToken(); var decorator = new PreTokenDecorator(); var argument = string.Empty; var tokenContent = new StringBuilder(); var frontMatterName = new StringBuilder(); var frontMatterValue = new StringBuilder(); var inFrontMatterToken = false; // Basic State Machine to parse the template input while (enumerator.IsEmpty == false) { switch (state) { case FlatTokenParserState.AtStart: ParseStart(enumerator, ref state); break; case FlatTokenParserState.InFrontMatter: ParseFrontMatter(enumerator, ref frontMatterName, ref state); break; case FlatTokenParserState.InFrontMatterComment: ParseFrontMatterComment(enumerator, ref state); break; case FlatTokenParserState.InFrontMatterOption: ParseFrontMatterOption(enumerator, ref frontMatterName, ref state, ref inFrontMatterToken, ref token); break; case FlatTokenParserState.InFrontMatterOptionValue: ParseFrontMatterOptionValue(preTemplate, enumerator, ref frontMatterName, ref frontMatterValue, ref state); break; case FlatTokenParserState.InPreamble: ParsePreamble(ref token, enumerator, ref state, ref tokenContent); break; case FlatTokenParserState.InTokenName: ParseTokenName(preTemplate, ref token, enumerator, ref state, ref inFrontMatterToken, ref tokenContent, preTemplate.Options); break; case FlatTokenParserState.InTokenValue: ParseTokenValue(preTemplate, ref token, enumerator, ref state, ref inFrontMatterToken, ref tokenContent, preTemplate.Options); break; case FlatTokenParserState.InTokenValueSingleQuotes: ParseTokenValueInSingleQuotes(enumerator, ref token, ref state, ref tokenContent); break; case FlatTokenParserState.InTokenValueDoubleQuotes: ParseTokenValueInDoubleQuotes(enumerator, ref token, ref state, ref tokenContent); break; case FlatTokenParserState.InTokenValueRunOff: ParseTokenValueRunOff(enumerator, ref preTemplate, ref token, ref state, ref inFrontMatterToken, ref tokenContent, preTemplate.Options); break; case FlatTokenParserState.InDecorator: ParseDecorator(preTemplate, ref token, enumerator, ref state, ref decorator, ref inFrontMatterToken, ref tokenContent, preTemplate.Options); break; case FlatTokenParserState.InDecoratorArgument: ParseDecoratorArgument(enumerator, ref state, ref decorator, ref argument, ref tokenContent); break; case FlatTokenParserState.InDecoratorArgumentSingleQuotes: ParseDecoratorArgumentInSingleQuotes(enumerator, ref state, ref decorator, ref argument, ref tokenContent); break; case FlatTokenParserState.InDecoratorArgumentDoubleQuotes: ParseDecoratorArgumentInDoubleQuotes(enumerator, ref state, ref decorator, ref argument, ref tokenContent); break; case FlatTokenParserState.InDecoratorArgumentRunOff: ParseDecoratorArgumentRunOff(enumerator, ref state, ref tokenContent); break; default: throw new TokenizerException($"Unknown FlatTokenParserState: {state}"); } } // Append current token if it has contents // Note: allow empty token values, as these will serve to truncate the last // token in the template if (string.IsNullOrWhiteSpace(token.Preamble) == false) { AppendToken(preTemplate, token, ref tokenContent, preTemplate.Options); } return(preTemplate); }
private void ParseFrontMatterOption(PreTokenEnumerator enumerator, ref StringBuilder frontMatterName, ref FlatTokenParserState state, ref bool inFrontMatterToken, ref PreToken token) { var next = enumerator.Next(); switch (next) { case ":": if (frontMatterName.ToString().Trim().ToLowerInvariant() == "set") { inFrontMatterToken = true; frontMatterName.Clear(); token.Location = enumerator.Location.Clone(); state = FlatTokenParserState.InTokenName; } else { state = FlatTokenParserState.InFrontMatterOptionValue; } break; default: frontMatterName.Append(next); break; } }