Exemple #1
0
        internal bool TryParseDictionary(ILookaroundEnumerator <Token> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode, AssignmentOperatorBehavior assignmentOperatorBehavior)
        {
            parsedNode = default;
            if (enumerator.Current.TokenType != TokenType.CurlyBraceOpen)
            {
                return(false);
            }

            var dictionaryItems = new Queue <DictionaryItemNode>();

            while (enumerator.MoveNext() && TryParseDictionaryItem(enumerator, out var dictionaryItem, assignmentOperatorBehavior))
            {
                dictionaryItems.Enqueue(dictionaryItem);
                if (enumerator.MoveNext() == false)
                {
                    throw new NotImplementedException();
                }
                if (enumerator.Current.TokenType != TokenType.Comma)
                {
                    break;
                }
            }

            if (enumerator.State == EnumeratorState.Complete || enumerator.Current.TokenType != TokenType.CurlyBraceClose)
            {
                throw new NotImplementedException();
            }
            parsedNode = new DictionaryNode(dictionaryItems);
            return(true);
        }
Exemple #2
0
 private bool TryParseDictionaryItem(ILookaroundEnumerator <Token> enumerator, [NotNullWhen(true)] out DictionaryItemNode?dictionaryItem, AssignmentOperatorBehavior assignmentOperatorBehavior)
 {
     dictionaryItem = default;
     if (TryParse(enumerator, out var key, assignmentOperatorBehavior) == false || key == default)
     {
         return(false);
     }
     if (enumerator.MoveNext() == false)
     {
         throw new NotImplementedException();
     }
     if (enumerator.Current.TokenType != TokenType.Colon)
     {
         throw new NotImplementedException();
     }
     if (enumerator.MoveNext() == false)
     {
         throw new NotImplementedException();
     }
     if (TryParse(enumerator, out var value, assignmentOperatorBehavior) == false || value == default)
     {
         throw new NotImplementedException();
     }
     dictionaryItem = new DictionaryItemNode(key, value);
     return(true);
 }
Exemple #3
0
        internal bool TryParse(ILookaroundEnumerator <Token> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode, AssignmentOperatorBehavior assignmentOperatorBehavior, int currentPrecedence = 0)
        {
            if (currentPrecedence == 0 && CustomParseDelegates != default)
            {
                var customResult = CustomParseDelegates.Select(del =>
                {
                    var Success = del(enumerator, out var Result, assignmentOperatorBehavior);
                    return(new { Success, Result });
                }).FirstOrDefault(res => res.Success)?.Result;
                if (customResult != default)
                {
                    enumerator.MoveNext(); // TODO: Is this right?
                    parsedNode = customResult;
                    return(true);
                }
            }

            if (currentPrecedence >= _PrecedenceGroups.Length)
            {
                return(TryParseBraces(enumerator, out parsedNode, assignmentOperatorBehavior));
            }

            return(_PrecedenceGroups[currentPrecedence].First().OperandCount switch
            {
                OperandCount.Binary => TryParseBinary(enumerator, currentPrecedence, out parsedNode, assignmentOperatorBehavior),
                OperandCount.Unary => TryParseUnary(enumerator, currentPrecedence, out parsedNode),
                _ => throw new NotImplementedException(),
            });
Exemple #4
0
 internal virtual bool TryReadSingleChar(ILookaroundEnumerator <char> enumerator, [NotNullWhen(true)] out Token?token)
 {
     token = LanguageDefinition.SingleCharTokens.TryGetValue(enumerator.Current, out var tokenType) ?
             new Token(tokenType, null, enumerator.Current) :
             default;
     return(token != default);
 }
        internal static bool TryParse(JinjaEnvironment environment, ILookaroundEnumerator <ParsingNode> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode)
        {
            parsedNode = default;
            string?ifClause   = null;
            string?elseClause = null;

            if (ExpressionNodeParser.Parser.TryParse(enumerator.Current, out var startWhiteSpace, out var endWhiteSpace) == false)
            {
                return(false);
            }
            if (ExpressionNodeParser.Parser.TryGetAccumulation(ExpressionNodeParser.ExpressionState.Expression, 0, out var expression) == false)
            {
                throw new NotImplementedException();
            }
            if (ExpressionNodeParser.Parser.TryGetAccumulation(ExpressionNodeParser.ExpressionState.IfClause, 0, out var ifClauseString))
            {
                ifClause = ifClauseString;
            }
            if (ExpressionNodeParser.Parser.TryGetAccumulation(ExpressionNodeParser.ExpressionState.ElseClause, 0, out var elseClauseString))
            {
                elseClause = elseClauseString;
            }
            if (string.IsNullOrEmpty(expression))
            {
                throw new NotImplementedException();
            }

            parsedNode = new ExpressionNode(environment, enumerator.Current, startWhiteSpace, endWhiteSpace, expression, ifClause, elseClause);
            return(true);
        }
Exemple #6
0
        private bool TryKeyword(ILookaroundEnumerator <char> enumerator, out Token?token)
        {
            token = default;
            var possibleKeywords = _KeywordLookups.Keys.OrderByDescending(keyword => keyword.Length);

            foreach (var possibleKeyword in possibleKeywords)
            {
                if (possibleKeyword[0] != enumerator.Current)
                {
                    continue;
                }
                var valid = true;
                for (int index = 1; index < possibleKeyword.Length; ++index)
                {
                    if (enumerator.TryGetNext(out var nextChar, index) == false || nextChar != possibleKeyword[index])
                    {
                        valid = false;
                        break;
                    }
                }
                if (valid)
                {
                    for (var i = 0; i < possibleKeyword.Length - 1; ++i)
                    {
                        enumerator.MoveNext();
                    }
                    token = new Token(_KeywordLookups[possibleKeyword], new string(possibleKeyword));
                    return(true);
                }
            }
            return(false);
        }
Exemple #7
0
        internal static bool TryParseRaw(JinjaEnvironment environment, Lexer lexer, ILookaroundEnumerator <ParsingNode> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode)
        {
            parsedNode = default;


            if (RawParser.StartBlock.TryParse(enumerator.Current) == false)
            {
                return(false);
            }
            var startParsingNode = enumerator.Current;
            var children         = new List <ParsingNode>();

            while (true)
            {
                if (enumerator.MoveNext() == false)
                {
                    throw new NotImplementedException();
                }
                if (RawParser.EndBlock.TryParse(enumerator.Current))
                {
                    break;
                }
                children.Add(enumerator.Current);
            }
            var endParsingNode = enumerator.Current;

            parsedNode = new RawNode(startParsingNode, children, endParsingNode);
            return(true);
        }
Exemple #8
0
 internal bool TryParse(ILookaroundEnumerator <Token> enumerator, [NotNullWhen(true)] out StateAction <TState>?action)
 {
     action = default;
     if (_ThrowAction != default)
     {
         action = _ThrowAction;
         return(true);
     }
     if (_Tokens.TryGetValue(enumerator.Current.TokenType, out var actionConditions))
     {
         foreach (var actionCondition in actionConditions)
         {
             if (actionCondition.Predicate(enumerator))
             {
                 action = actionCondition.Action;
                 return(true);
             }
         }
     }
     if (_ElseAction != default)
     {
         action = _ElseAction;
         return(true);
     }
     return(false);
 }
Exemple #9
0
 internal virtual bool TryReadStringLiteral(ILookaroundEnumerator <char> enumerator, [NotNullWhen(true)] out Token?token)
 {
     token = default;
     if (enumerator.Current != '"')
     {
         return(false);
     }
     throw new NotImplementedException();
 }
Exemple #10
0
 internal bool TryParseTuple(ILookaroundEnumerator <Token> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode, AssignmentOperatorBehavior assignmentOperatorBehavior)
 {
     if (TryParseCommaSeperatedSet(enumerator, TokenType.Operator, _OPERATOR_PAREN_OPEN, ParenClose, out var parsedListItems, minimumItems: 2, assignmentOperatorBehavior))
     {
         parsedNode = new TupleNode(parsedListItems);
         return(true);
     }
     parsedNode = default;
     return(false);
 }
Exemple #11
0
 internal bool TryParseList(ILookaroundEnumerator <Token> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode, AssignmentOperatorBehavior assignmentOperatorBehavior)
 {
     if (TryParseCommaSeperatedSet(enumerator, TokenType.Operator, _OPERATOR_SQUARE_BRACE_OPEN, SquareBraceClose, out var parsedListItems, minimumItems: 0, assignmentOperatorBehavior))
     {
         parsedNode = new ListNode(parsedListItems);
         return(true);
     }
     parsedNode = default;
     return(false);
 }
Exemple #12
0
        private static IEnumerable <Token> ReadUntil(ILookaroundEnumerator <Token> enumerator, params TokenType[] stopTokens)
        {
            do
            {
                yield return(enumerator.Current);

                if (stopTokens.Contains(enumerator.Current.TokenType))
                {
                    yield break;
                }
            } while (enumerator.MoveNext());
        }
Exemple #13
0
        internal static WhiteSpaceNode Parse(ILookaroundEnumerator <ParsingNode> enumerator)
        {
            var nodes = new Queue <ParsingNode>();

            nodes.Enqueue(enumerator.Current);

            while (enumerator.TryGetNext(out var nextNode) && nextNode.NodeType == ParsingNodeType.WhiteSpace)
            {
                enumerator.MoveNext();
                nodes.Enqueue(nextNode);
            }
            return(new WhiteSpaceNode(nodes));
        }
Exemple #14
0
 internal static bool TryParse(JinjaEnvironment environment, Lexer lexer, ILookaroundEnumerator <ParsingNode> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode)
 {
     parsedNode = _Delegates.Select(del =>
     {
         var Result = del(environment, lexer, enumerator, out var ParsedNode);
         return(new
         {
             Result,
             ParsedNode,
         });
     }).FirstOrDefault(res => res.Result)?.ParsedNode;
     return(parsedNode != default);
 }
Exemple #15
0
        internal virtual bool TryReadOperator(ILookaroundEnumerator <char> enumerator, [NotNullWhen(true)] out Token?token)
        {
            token = default;
            var initialPossibleOperators = _Operators.Keys.Where(operatorArr => operatorArr[0] == enumerator.Current).ToArray();

            if (initialPossibleOperators.Length == 0)
            {
                return(false);
            }

            List <char[]> possibleOperators = new List <char[]>();

            foreach (var op in initialPossibleOperators.OrderByDescending(x => x.Length))
            {
                if (op[op.Length - 1].IsLetter())
                {
                    // If the operator ends in a letter (like "is") - then the *NEXT* char after it should _NOT_ be a letter.
                    if (enumerator.TryGetNext(out var nextChar, op.Length) && nextChar.IsLetter())
                    {
                        continue; // Skip this operator - the next character is a letter, and can't properly terminate the operator
                    }
                }

                var valid = true;
                for (var index = op.Length - 1; index > 0; --index)
                {
                    if ((enumerator.TryGetNext(out var nextChar, index) == false) || nextChar != op[index])
                    {
                        valid = false;
                        break;
                    }
                }
                if (valid == false)
                {
                    continue;
                }
                if (enumerator.Current != op[0])
                {
                    continue;
                }
                possibleOperators.Add(op);
            }
            if (possibleOperators.Count == 0)
            {
                return(false);
            }
            possibleOperators = possibleOperators.Distinct(CharArrayEqualityComparer.Instance).ToList();
            token             = new Token(TokenType.Operator, _Operators[possibleOperators[0]].SecondaryTokenType, _Operators[possibleOperators[0]].Text);
            enumerator.MoveNext(possibleOperators[0].Length - 1);
            return(true);
        }
Exemple #16
0
        internal static bool TryParseFor(JinjaEnvironment environment, Lexer lexer, ILookaroundEnumerator <ParsingNode> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode)
        {
            ContainerNode? elseBlock  = null;
            ExpressionNode?filterNode = null;

            parsedNode = default;

            if (ForParser.StartBlock.TryParse(enumerator.Current, out var outsideStart, out var primaryInsideStart) == false)
            {
                return(false);
            }
            var primaryStartParsingNode = enumerator.Current;

            if (ForParser.StartBlock.TryGetAccumulations(ForParser.ForState.VariableNames, out var variableNames) == false || variableNames.Length == 0)
            {
                throw new NotImplementedException();
            }
            if (ForParser.StartBlock.TryGetAccumulation(ForParser.ForState.Expression, 0, out var expression) == false)
            {
                throw new NotImplementedException();
            }
            if (ForParser.StartBlock.TryGetAccumulation(ForParser.ForState.Filter, 0, out string?filter))
            {
                filterNode = ExpressionNode.FromString(environment, filter);
            }
            ForParser.StartBlock.TryGetVariable <bool>("recursive", out var recursive);
            enumerator.MoveNext();
            var primaryBlockChildren = ASTGenerator.ParseUntilFailure(environment, lexer, enumerator).ToArray();

            ContainerNode primaryBlock;

            if (ForParser.ElseBlock.TryParse(enumerator.Current, out var primaryInsideEnd, out var elseInsideStart))
            {
                var elseStartParsingNode = enumerator.Current;
                enumerator.MoveNext();
                var elseBlockChildren = ASTGenerator.ParseUntilFailure(environment, lexer, enumerator).ToArray();
                if (ForParser.EndBlock.TryParse(enumerator.Current, out var elseInsideEnd, out var outsideEnd) == false)
                {
                    throw new NotImplementedException();
                }

                primaryBlock = new ContainerNode(primaryStartParsingNode, primaryBlockChildren, null,
                                                 new WhiteSpaceControlSet(primaryInsideStart, primaryInsideEnd));
                elseBlock = new ContainerNode(elseStartParsingNode, elseBlockChildren, null,
                                              new WhiteSpaceControlSet(elseInsideStart, elseInsideEnd));
                parsedNode = new ForNode(primaryBlock, elseBlock, variableNames, ExpressionNode.FromString(environment, expression),
                                         filterNode, recursive, enumerator.Current, new WhiteSpaceControlSet(outsideStart, outsideEnd));
                return(true);
            }
Exemple #17
0
        private static IEnumerable <Token> ReadWhile(ILookaroundEnumerator <Token> enumerator, Func <TokenType, bool> predicate)
        {
            yield return(enumerator.Current);

            while (enumerator.TryGetNext(out var nextToken) && predicate(nextToken.TokenType))
            {
                var moveNext = enumerator.MoveNext();
                yield return(enumerator.Current);

                if (moveNext == false)
                {
                    yield break;
                }
            }
        }
Exemple #18
0
        private bool TryParseCommaSeperatedSet(ILookaroundEnumerator <Token> enumerator, TokenType startTokenType, string?startTokenText, TokenType endTokenType, [NotNullWhen(true)] out IEnumerable <ASTNode>?parsedNodes, int minimumItems, AssignmentOperatorBehavior assignmentOperatorBehavior)
        {
            parsedNodes = default;
            if (enumerator.Current.TokenType != startTokenType)
            {
                return(false);
            }
            if (startTokenText != null && enumerator.Current.TextValue != startTokenText)
            {
                return(false);
            }

            if (enumerator.TryGetPrevious(out var prevToken) == true)
            {
                if (prevToken.TokenType.IsTerminal())
                {
                    return(false);
                }
            }
            enumerator.MoveNext();
            var queue = new Queue <ASTNode>();

            while (TryParse(enumerator, out var listItem, assignmentOperatorBehavior))
            {
                queue.Enqueue(listItem);
                if (enumerator.Current.TokenType != TokenType.Comma)
                {
                    break;
                }
                if (enumerator.MoveNext() == false)
                {
                    throw new ParseException(ExpressionParserStrings.ResourceManager.GetString("ParsingError_UnterminatedCollectionLiteral", CultureInfo.InvariantCulture));
                }
            }
            if (queue.Count < minimumItems)
            {
                throw new NotImplementedException();
            }

            if (enumerator.Current.TokenType != endTokenType)
            {
                throw new NotImplementedException();
            }
            parsedNodes = queue;
            return(true);
        }
Exemple #19
0
        internal virtual bool TryReadIdentifier(ILookaroundEnumerator <char> enumerator, [NotNullWhen(true)] out Token?token)
        {
            token = default;
            if (enumerator.Current.IsLetter() == false && enumerator.Current != '_')
            {
                return(false);
            }
            using var checkoutRecord = StringBuilderPool.Instance.Checkout();
            var stringBuilder = checkoutRecord.CheckedOutObject;

            stringBuilder.Append(enumerator.Current);
            while (enumerator.TryGetNext(out var nextChar) && nextChar.IsLetter() || nextChar.IsDigit() || nextChar == '_')
            {
                stringBuilder.Append(enumerator.MoveNextAndGetValue(out _));
            }
            token = new Token(TokenType.Identifier, null, stringBuilder);
            return(true);
        }
Exemple #20
0
        private bool TryNewLine(ILookaroundEnumerator <char> enumerator, out Token?token)
        {
            token = default;
            switch (enumerator.Current)
            {
            case '\r':
                if (enumerator.TryGetNext(out var nextChar) && nextChar == '\n')
                {
                    enumerator.MoveNext();
                    token = Token.NewLine;
                    return(true);
                }
                break;

            case '\n':
                token = Token.NewLine;
                return(true);
            }
            return(false);
        }
Exemple #21
0
        internal override bool TryReadStringLiteral(ILookaroundEnumerator <char> enumerator, [NotNullWhen(true)] out Token?token)
        {
            token = default;
            if (enumerator.Current != '"' && enumerator.Current != '\'')
            {
                return(false);
            }
            using var checkoutRecord = StringBuilderPool.Instance.Checkout();
            var stringBuilder = checkoutRecord.CheckedOutObject;

            var quoteChar = enumerator.Current;

            while (enumerator.MoveNext() && enumerator.Current != quoteChar)
            {
                if (enumerator.Current == '\\')
                {
                    if (enumerator.TryGetNext(out var nextChar) == false)
                    {
                        throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_UnrecognizedEscape", CultureInfo.InvariantCulture));
                    }
                    if (nextChar.IsValidEscapedChar() == false)
                    {
                        throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_UnrecognizedEscape", CultureInfo.InvariantCulture));
                    }
                    enumerator.MoveNext(); //Eat the backslash
                    stringBuilder.Append(enumerator.Current.Escape());
                    continue;
                }
                stringBuilder.Append(enumerator.Current);
            }
            if (enumerator.State == EnumeratorState.Complete)
            {
                throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_NewlineInConstant", CultureInfo.InvariantCulture));
            }
            if (enumerator.Current != quoteChar)
            {
                throw new LexingException($"Expected {quoteChar} : Encountered {enumerator.Current}");
            }
            token = new Token(TokenType.StringLiteral, null, stringBuilder);
            return(true);
        }
Exemple #22
0
        internal static bool TryParseSet(JinjaEnvironment environment, Lexer lexer, ILookaroundEnumerator <ParsingNode> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode)
        {
            parsedNode = default;
            if (SetParser.StartBlock.TryParse(enumerator.Current) == false)
            {
                return(false);
            }
            var startParsingNode = enumerator.Current;

            if (SetParser.StartBlock.TryGetAccumulations(SetParser.SetState.VariableName, out var variableNames) == false)
            {
                throw new NotImplementedException();
            }
            if (variableNames == null || variableNames.Length == 0)
            {
                throw new NotImplementedException();
            }
            if (variableNames.Length != 1)
            {
                throw new NotImplementedException();                            // Not supported yet
            }
            if (SetParser.StartBlock.TryGetAccumulation(SetParser.SetState.AssignmentExpression, 0, out var assignmentExpression))
            {
                parsedNode = new SetNode(startParsingNode, variableNames, assignmentExpression, null);
                return(true);
            }

            enumerator.MoveNext();
            var contents = ASTGenerator.ParseUntilFailure(environment, lexer, enumerator).ToArray();


            if (SetParser.EndBlock.TryParse(enumerator.Current) == false)
            {
                throw new NotImplementedException();
            }

            var contentsNode = new ContainerNode(null, contents, null);

            parsedNode = new SetNode(startParsingNode, variableNames, contentsNode, enumerator.Current);
            return(true);
        }
Exemple #23
0
        internal static bool TryParseCall(JinjaEnvironment environment, Lexer lexer, ILookaroundEnumerator <ParsingNode> enumerator, [NotNullWhen(true)] out ASTNode?parsedNode)
        {
            parsedNode = default;


            if (CallParser.StartBlock.TryParse(enumerator.Current, out var outsideStart, out var insideStart) == false)
            {
                return(false);
            }
            if (CallParser.StartBlock.TryGetAccumulation(CallParser.CallState.CallDefinition, 0, out var callDefinition) == false)
            {
                throw new NotImplementedException();
            }
            var startParsingNode = enumerator.Current;

            enumerator.MoveNext();
            var contents = ASTGenerator.ParseUntilFailure(environment, lexer, enumerator).ToArray();

            if (CallParser.EndBlock.TryParse(enumerator.Current, out var insideEnd, out var outsideEnd) == false)
            {
                return(false);
            }
            var endParsingNode = enumerator.Current;
            var contentsNode   = new ContainerNode(null, contents, null,
                                                   new WhiteSpaceControlSet(insideStart, insideEnd)
                                                   );

            if (TryParseCallDefinition(lexer, callDefinition, out var callArgumentList, out var macroCall) == false)
            {
                throw new NotImplementedException();
            }


            parsedNode = new CallNode(startParsingNode,
                                      ExpressionNode.FromString(environment, callArgumentList),
                                      ExpressionNode.FromString(environment, macroCall), contentsNode, endParsingNode,
                                      new WhiteSpaceControlSet(outsideStart, outsideEnd)
                                      );
            return(true);
        }
Exemple #24
0
        internal virtual bool TryReadCharacterLiteral(ILookaroundEnumerator <char> enumerator, [NotNullWhen(true)] out Token?token)
        {
            token = default;
            var escaped = false;

            if (enumerator.Current != '\'')
            {
                return(false);
            }
            enumerator.MoveNext(); //Eat the quote.
            if (enumerator.TryGetNext(out var nextChar) == false)
            {
                throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_NewlineInConstant", CultureInfo.InvariantCulture));
            }
            if (enumerator.Current == '\'')
            {
                throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_EmptyCharacterLiteral", CultureInfo.InvariantCulture));
            }
            if (enumerator.Current == '\\')
            {
                if (nextChar.IsValidEscapedChar() == false)
                {
                    throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_UnrecognizedEscape", CultureInfo.InvariantCulture));
                }
                enumerator.MoveNext(); // Pass the backslash
                escaped = true;
            }
            if (enumerator.TryGetNext(out nextChar) == false)
            {
                throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_NewlineInConstant", CultureInfo.InvariantCulture));
            }
            if (nextChar != '\'')
            {
                throw new LexingException(ExpressionParserStrings.ResourceManager.GetString("LexerError_TooManyCharsInCharLiteral", CultureInfo.InvariantCulture));
            }
            token = new Token(TokenType.CharacterLiteral, null, escaped ? enumerator.Current.Escape() : enumerator.Current);
            enumerator.MoveNext(); //Move to quote
            return(true);
        }
Exemple #25
0
        internal static bool TryParseLineStatement(ILookaroundEnumerator <Token> enumerator, [NotNullWhen(true)] out ParsingNode?parsedNode)
        {
            parsedNode = default;
            if (enumerator.Current.TokenType != TokenType.LineStatement)
            {
                return(false);
            }
            var continueLoop = true;
            var queue        = new Queue <Token>();
            var nestingStack = new Stack <TokenType>();

            while (continueLoop)
            {
                if (enumerator.Current.TokenType == TokenType.NewLine && nestingStack.Count > 0)
                {
                    enumerator.MoveNext();
                    continue;
                }
                else if (enumerator.Current.TokenType.IsOpeningBrace())
                {
                    nestingStack.Push(enumerator.Current.TokenType);
                }
                else if (enumerator.Current.TokenType.IsClosingBrace())
                {
                    if (nestingStack.Peek().IsMatchingBrace(enumerator.Current.TokenType))
                    {
                        nestingStack.Pop();
                    }
                    else
                    {
                        throw new NotImplementedException(); //Unbalanced stack
                    }
                }

                if (enumerator.TryGetNext(out var nextToken) == false)
                {
                    continueLoop = false;
                }
Exemple #26
0
        private bool TryWhiteSpace(ILookaroundEnumerator <char> enumerator, out Token?token)
        {
            token = default;
            if (enumerator.Current.IsWhiteSpace() == false)
            {
                return(false);
            }
            var queue = new Queue <char>();

            queue.Enqueue(enumerator.Current);

            while (enumerator.TryGetNext(out var nextChar) && nextChar.IsWhiteSpace() && nextChar.IsNotNewLine())
            {
                var result = enumerator.MoveNext();
                queue.Enqueue(enumerator.Current);
                if (result == false)
                {
                    break;
                }
            }
            token = new Token(TokenType.WhiteSpace, queue);
            return(true);
        }
Exemple #27
0
        internal virtual bool TryReadNumericLiteral(ILookaroundEnumerator <char> enumerator, [NotNullWhen(true)] out Token?token)
        {
            token = default;
            if (enumerator.Current.IsDigit() == false)
            {
                return(false);
            }
            using var checkoutRecord = StringBuilderPool.Instance.Checkout();
            var stringBuilder = checkoutRecord.CheckedOutObject;

            stringBuilder.Append(enumerator.Current);
            var hasDecimal = false;

            // Get the portion before the decimal point...
            while (enumerator.TryGetNext(out var nextChar) && nextChar.IsDigit())
            {
                stringBuilder.Append(enumerator.MoveNextAndGetValue(out _));
            }

            // Attempt to get the decimal point
            if (enumerator.TryGetNext(out var decimalPoint) && decimalPoint == '.' &&
                enumerator.TryGetNext(out var decimalDigit, 2) && decimalDigit.IsDigit())
            {
                hasDecimal = true;
                stringBuilder.Append(enumerator.MoveNextAndGetValue(out _));
            }

            // Get the portion after the decimal point...
            while (hasDecimal && enumerator.TryGetNext(out var nextChar) && nextChar.IsDigit())
            {
                stringBuilder.Append(enumerator.MoveNextAndGetValue(out _));
            }

            token = new Token(hasDecimal ? TokenType.FloatingLiteral : TokenType.IntegerLiteral, null, stringBuilder);
            return(true);
        }
Exemple #28
0
 private static IEnumerable <Token> ReadOne(ILookaroundEnumerator <Token> enumerator)
 {
     yield return(enumerator.Current);
 }
Exemple #29
0
 private static ParsingNode LineStatementOrUnknown(bool canBeLineStatement, ILookaroundEnumerator <Token> enumerator)
 {
     return((canBeLineStatement && TryParseLineStatement(enumerator, out var lineStatementNode)) ?
            lineStatementNode :
            new ParsingNode(ParsingNodeType.Output, ReadOne(enumerator)));
 }
Exemple #30
0
        internal static IEnumerable <ASTNode> ParseUntilFailure(JinjaEnvironment environment, Lexer lexer, ILookaroundEnumerator <ParsingNode> enumerator)
        {
            do
            {
                ASTNode?astNode = default;
                switch (enumerator.Current.NodeType)
                {
                case ParsingNodeType.Statement:
                    StatementNode.TryParse(environment, lexer, enumerator, out astNode);
                    break;

                case ParsingNodeType.NewLine:
                    astNode = new NewLineNode(enumerator.Current);
                    break;

                case ParsingNodeType.Comment:
                    astNode = new CommentNode(enumerator.Current);
                    break;

                case ParsingNodeType.WhiteSpace:
                    astNode = WhiteSpaceNode.Parse(enumerator);
                    break;

                case ParsingNodeType.Expression:
                    if (ExpressionNode.TryParse(environment, enumerator, out astNode) == false)
                    {
                        throw new NotImplementedException();
                    }
                    break;

                case ParsingNodeType.Output:
                    astNode = new OutputNode(enumerator.Current);
                    break;
                }
                if (astNode == default)
                {
                    yield break;
                }
                yield return(astNode);
            } while (enumerator.MoveNext());
        }