private List <SyntaxToken> ExpandNestedMacro(NestedMacroExpansionLexer lexer)
        {
            var result = new List <SyntaxToken>();

            SyntaxToken token;

            while ((token = lexer.GetNextToken()) != null)
            {
                // Do token pasting (##).
                if (lexer.Peek(0) != null && lexer.Peek(0).Kind == SyntaxKind.HashHashToken &&
                    lexer.Peek(1) != null && lexer.Peek(1).Kind != SyntaxKind.HashHashToken)
                {
                    lexer.GetNextToken();
                    var concatenatedText = token.Text + lexer.GetNextToken().Text;
                    token = new HlslLexer(new StringText(concatenatedText)).Lex(LexerMode.Syntax);
                }

                List <SyntaxToken> expandedTokens;
                if (TryExpandMacro(token, lexer, out expandedTokens))
                {
                    result.AddRange(expandedTokens);
                }
                else
                {
                    result.Add(token);
                }
            }

            return(result);
        }
Example #2
0
        public static IReadOnlyList<SyntaxToken> ParseAllTokens(SourceText sourceText, IIncludeFileSystem fileSystem = null)
        {
            var tokens = new List<SyntaxToken>();

            var lexer = new HlslLexer(sourceText, fileSystem: fileSystem);
            SyntaxToken token;
            do
            {
                tokens.Add(token = lexer.Lex(LexerMode.Syntax));
            } while (token.Kind != SyntaxKind.EndOfFileToken);

            return tokens;
        }
Example #3
0
        private static SyntaxTree Parse(SourceText sourceText, ParserOptions options, IIncludeFileSystem fileSystem, Func<HlslParser, SyntaxNode> parseFunc)
        {
            var lexer = new HlslLexer(sourceText, options, fileSystem);
            var parser = new HlslParser(lexer);

            var result = new SyntaxTree(sourceText,
                syntaxTree => new Tuple<SyntaxNode, List<FileSegment>>(
                    parseFunc(parser),
                    lexer.FileSegments));

            Debug.WriteLine(DateTime.Now +  " - Finished parsing");

            return result;
        }
 public DirectiveParser(HlslLexer lexer, DirectiveStack directiveStack)
     : base(lexer, LexerMode.Directive)
 {
     _lexer = lexer;
     _directiveStack = directiveStack;
 }
Example #5
0
 public DirectiveParser(HlslLexer lexer, DirectiveStack directiveStack)
     : base(lexer, LexerMode.Directive)
 {
     _lexer          = lexer;
     _directiveStack = directiveStack;
 }
 public BaseMacroExpansionLexer(HlslLexer lexer)
 {
     _lexer = lexer;
 }
 public BaseMacroExpansionLexer(HlslLexer lexer)
 {
     _lexer = lexer;
 }
        private List<SyntaxToken> ExpandNestedMacro(NestedMacroExpansionLexer lexer)
        {
            var result = new List<SyntaxToken>();

            SyntaxToken token;
            while ((token = lexer.GetNextToken()) != null)
            {
                // Do token pasting (##).
                if (lexer.Peek(0) != null && lexer.Peek(0).Kind == SyntaxKind.HashHashToken
                    && lexer.Peek(1) != null && lexer.Peek(1).Kind != SyntaxKind.HashHashToken)
                {
                    lexer.GetNextToken();
                    var concatenatedText = token.Text + lexer.GetNextToken().Text;
                    token = new HlslLexer(new StringText(concatenatedText)).Lex(LexerMode.Syntax);
                }

                List<SyntaxToken> expandedTokens;
                if (TryExpandMacro(token, lexer, out expandedTokens))
                    result.AddRange(expandedTokens);
                else
                    result.Add(token);
            }

            return result;
        }
        private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List<SyntaxToken> expandedTokens)
        {
            expandedTokens = null;

            // First, check if this token might be a macro.
            DefineDirectiveTriviaSyntax directive;
            if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined)
                return false;

            // Check that this macro is not disabled.
            if (_currentlyExpandingMacros.Contains(directive))
                return false;

            MacroReference macroReference;
            List<SyntaxToken> macroBody;
            SyntaxToken lastToken;
            switch (directive.Kind)
            {
                case SyntaxKind.FunctionLikeDefineDirectiveTrivia:
                    // For function-like macros, check for, and expand, macro arguments.
                    var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax) directive;

                    // ... check to see if the next token is an open paren.
                    if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken)
                        return false;

                    // If it is, then parse the macro arguments, and
                    // check that we have the correct number of arguments.
                    ExpandMacros = false;
                    var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList();
                    ExpandMacros = true;

                    if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count)
                    {
                        expandedTokens = new List<SyntaxToken>
                        {
                            token
                                .WithDiagnostic(Diagnostic.Format(token.Span, DiagnosticId.NotEnoughMacroParameters, token.Text))
                                .WithTrailingTrivia(new[] { macroArguments })
                        };
                        return true;
                    }

                    var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax) directive;
                    macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective);

                    // Expand arguments.
                    var expandedArguments = macroArguments.Arguments
                        .Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList())
                        .ToList();

                    // Replace parameters with possibly-expanded arguments.
                    macroBody = ReplaceParameters(
                        macroArguments.Arguments.ToList(), expandedArguments,
                        functionLikeDefineDirective.Parameters,
                        functionLikeDefineDirective.Body);

                    lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token;

                    break;

                case SyntaxKind.ObjectLikeDefineDirectiveTrivia:
                    macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax) directive);
                    macroBody = directive.MacroBody;
                    lastToken = token;
                    break;

                default:
                    throw new ArgumentOutOfRangeException();
            }

            // Push the current macro onto the stack - this prevents recursive expansion.
            _currentlyExpandingMacros.Push(directive);

            // Scan macro body for nested macros.
            expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody));

            // Relex identifier tokens, because at this point keywords are stored as identifiers.
            for (var i = 0; i < expandedTokens.Count; i++)
                if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken)
                {
                    var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax);
                    expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind);
                }

            var localExpandedTokens = expandedTokens;
            expandedTokens = expandedTokens
                .Select((x, i) =>
                {
                    var result = x
                        .WithOriginalMacroReference(macroReference, i == 0)
                        .WithSpan(macroReference.SourceRange, macroReference.Span);
                    if (i == 0)
                        result = result.WithLeadingTrivia(token.LeadingTrivia);
                    if (i == localExpandedTokens.Count - 1)
                        result = result.WithTrailingTrivia(lastToken.TrailingTrivia);
                    return result;
                })
                .ToList();

            _currentlyExpandingMacros.Pop();

            return true;
        }
Example #10
0
        private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List <SyntaxToken> expandedTokens)
        {
            expandedTokens = null;

            // First, check if this token might be a macro.
            DefineDirectiveTriviaSyntax directive;

            if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined)
            {
                return(false);
            }

            // Check that this macro is not disabled.
            if (_currentlyExpandingMacros.Contains(directive))
            {
                return(false);
            }

            MacroReference     macroReference;
            List <SyntaxToken> macroBody;
            SyntaxToken        lastToken;

            switch (directive.Kind)
            {
            case SyntaxKind.FunctionLikeDefineDirectiveTrivia:
                // For function-like macros, check for, and expand, macro arguments.
                var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax)directive;

                // ... check to see if the next token is an open paren.
                if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken)
                {
                    return(false);
                }

                // If it is, then parse the macro arguments, and
                // check that we have the correct number of arguments.
                ExpandMacros = false;
                var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList();
                ExpandMacros = true;

                if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count)
                {
                    expandedTokens = new List <SyntaxToken>
                    {
                        token
                        .WithDiagnostic(Diagnostic.Format(token.Span, DiagnosticId.NotEnoughMacroParameters, token.Text))
                        .WithTrailingTrivia(new[] { macroArguments })
                    };
                    return(true);
                }

                var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax)directive;
                macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective);

                // Expand arguments.
                var expandedArguments = macroArguments.Arguments
                                        .Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList())
                                        .ToList();

                // Replace parameters with possibly-expanded arguments.
                macroBody = ReplaceParameters(
                    macroArguments.Arguments.ToList(), expandedArguments,
                    functionLikeDefineDirective.Parameters,
                    functionLikeDefineDirective.Body);

                lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token;

                break;

            case SyntaxKind.ObjectLikeDefineDirectiveTrivia:
                macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax)directive);
                macroBody      = directive.MacroBody;
                lastToken      = token;
                break;

            default:
                throw new ArgumentOutOfRangeException();
            }

            // Push the current macro onto the stack - this prevents recursive expansion.
            _currentlyExpandingMacros.Push(directive);

            // Scan macro body for nested macros.
            expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody));

            // Relex identifier tokens, because at this point keywords are stored as identifiers.
            for (var i = 0; i < expandedTokens.Count; i++)
            {
                if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken)
                {
                    var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax);
                    expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind);
                }
            }

            var localExpandedTokens = expandedTokens;

            expandedTokens = expandedTokens
                             .Select((x, i) =>
            {
                var result = x
                             .WithOriginalMacroReference(macroReference, i == 0)
                             .WithSpan(macroReference.SourceRange, macroReference.Span);
                if (i == 0)
                {
                    result = result.WithLeadingTrivia(token.LeadingTrivia);
                }
                if (i == localExpandedTokens.Count - 1)
                {
                    result = result.WithTrailingTrivia(lastToken.TrailingTrivia);
                }
                return(result);
            })
                             .ToList();

            _currentlyExpandingMacros.Pop();

            return(true);
        }