private List <SyntaxToken> ExpandNestedMacro(NestedMacroExpansionLexer lexer) { var result = new List <SyntaxToken>(); SyntaxToken token; while ((token = lexer.GetNextToken()) != null) { // Do token pasting (##). if (lexer.Peek(0) != null && lexer.Peek(0).Kind == SyntaxKind.HashHashToken && lexer.Peek(1) != null && lexer.Peek(1).Kind != SyntaxKind.HashHashToken) { lexer.GetNextToken(); var concatenatedText = token.Text + lexer.GetNextToken().Text; token = new HlslLexer(new StringText(concatenatedText)).Lex(LexerMode.Syntax); } List <SyntaxToken> expandedTokens; if (TryExpandMacro(token, lexer, out expandedTokens)) { result.AddRange(expandedTokens); } else { result.Add(token); } } return(result); }
public static IReadOnlyList<SyntaxToken> ParseAllTokens(SourceText sourceText, IIncludeFileSystem fileSystem = null) { var tokens = new List<SyntaxToken>(); var lexer = new HlslLexer(sourceText, fileSystem: fileSystem); SyntaxToken token; do { tokens.Add(token = lexer.Lex(LexerMode.Syntax)); } while (token.Kind != SyntaxKind.EndOfFileToken); return tokens; }
private static SyntaxTree Parse(SourceText sourceText, ParserOptions options, IIncludeFileSystem fileSystem, Func<HlslParser, SyntaxNode> parseFunc) { var lexer = new HlslLexer(sourceText, options, fileSystem); var parser = new HlslParser(lexer); var result = new SyntaxTree(sourceText, syntaxTree => new Tuple<SyntaxNode, List<FileSegment>>( parseFunc(parser), lexer.FileSegments)); Debug.WriteLine(DateTime.Now + " - Finished parsing"); return result; }
public DirectiveParser(HlslLexer lexer, DirectiveStack directiveStack) : base(lexer, LexerMode.Directive) { _lexer = lexer; _directiveStack = directiveStack; }
public BaseMacroExpansionLexer(HlslLexer lexer) { _lexer = lexer; }
private List<SyntaxToken> ExpandNestedMacro(NestedMacroExpansionLexer lexer) { var result = new List<SyntaxToken>(); SyntaxToken token; while ((token = lexer.GetNextToken()) != null) { // Do token pasting (##). if (lexer.Peek(0) != null && lexer.Peek(0).Kind == SyntaxKind.HashHashToken && lexer.Peek(1) != null && lexer.Peek(1).Kind != SyntaxKind.HashHashToken) { lexer.GetNextToken(); var concatenatedText = token.Text + lexer.GetNextToken().Text; token = new HlslLexer(new StringText(concatenatedText)).Lex(LexerMode.Syntax); } List<SyntaxToken> expandedTokens; if (TryExpandMacro(token, lexer, out expandedTokens)) result.AddRange(expandedTokens); else result.Add(token); } return result; }
private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List<SyntaxToken> expandedTokens) { expandedTokens = null; // First, check if this token might be a macro. DefineDirectiveTriviaSyntax directive; if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined) return false; // Check that this macro is not disabled. if (_currentlyExpandingMacros.Contains(directive)) return false; MacroReference macroReference; List<SyntaxToken> macroBody; SyntaxToken lastToken; switch (directive.Kind) { case SyntaxKind.FunctionLikeDefineDirectiveTrivia: // For function-like macros, check for, and expand, macro arguments. var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax) directive; // ... check to see if the next token is an open paren. if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken) return false; // If it is, then parse the macro arguments, and // check that we have the correct number of arguments. ExpandMacros = false; var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList(); ExpandMacros = true; if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count) { expandedTokens = new List<SyntaxToken> { token .WithDiagnostic(Diagnostic.Format(token.Span, DiagnosticId.NotEnoughMacroParameters, token.Text)) .WithTrailingTrivia(new[] { macroArguments }) }; return true; } var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax) directive; macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective); // Expand arguments. var expandedArguments = macroArguments.Arguments .Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList()) .ToList(); // Replace parameters with possibly-expanded arguments. macroBody = ReplaceParameters( macroArguments.Arguments.ToList(), expandedArguments, functionLikeDefineDirective.Parameters, functionLikeDefineDirective.Body); lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token; break; case SyntaxKind.ObjectLikeDefineDirectiveTrivia: macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax) directive); macroBody = directive.MacroBody; lastToken = token; break; default: throw new ArgumentOutOfRangeException(); } // Push the current macro onto the stack - this prevents recursive expansion. _currentlyExpandingMacros.Push(directive); // Scan macro body for nested macros. expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody)); // Relex identifier tokens, because at this point keywords are stored as identifiers. for (var i = 0; i < expandedTokens.Count; i++) if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken) { var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax); expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind); } var localExpandedTokens = expandedTokens; expandedTokens = expandedTokens .Select((x, i) => { var result = x .WithOriginalMacroReference(macroReference, i == 0) .WithSpan(macroReference.SourceRange, macroReference.Span); if (i == 0) result = result.WithLeadingTrivia(token.LeadingTrivia); if (i == localExpandedTokens.Count - 1) result = result.WithTrailingTrivia(lastToken.TrailingTrivia); return result; }) .ToList(); _currentlyExpandingMacros.Pop(); return true; }
private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List <SyntaxToken> expandedTokens) { expandedTokens = null; // First, check if this token might be a macro. DefineDirectiveTriviaSyntax directive; if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined) { return(false); } // Check that this macro is not disabled. if (_currentlyExpandingMacros.Contains(directive)) { return(false); } MacroReference macroReference; List <SyntaxToken> macroBody; SyntaxToken lastToken; switch (directive.Kind) { case SyntaxKind.FunctionLikeDefineDirectiveTrivia: // For function-like macros, check for, and expand, macro arguments. var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax)directive; // ... check to see if the next token is an open paren. if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken) { return(false); } // If it is, then parse the macro arguments, and // check that we have the correct number of arguments. ExpandMacros = false; var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList(); ExpandMacros = true; if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count) { expandedTokens = new List <SyntaxToken> { token .WithDiagnostic(Diagnostic.Format(token.Span, DiagnosticId.NotEnoughMacroParameters, token.Text)) .WithTrailingTrivia(new[] { macroArguments }) }; return(true); } var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax)directive; macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective); // Expand arguments. var expandedArguments = macroArguments.Arguments .Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList()) .ToList(); // Replace parameters with possibly-expanded arguments. macroBody = ReplaceParameters( macroArguments.Arguments.ToList(), expandedArguments, functionLikeDefineDirective.Parameters, functionLikeDefineDirective.Body); lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token; break; case SyntaxKind.ObjectLikeDefineDirectiveTrivia: macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax)directive); macroBody = directive.MacroBody; lastToken = token; break; default: throw new ArgumentOutOfRangeException(); } // Push the current macro onto the stack - this prevents recursive expansion. _currentlyExpandingMacros.Push(directive); // Scan macro body for nested macros. expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody)); // Relex identifier tokens, because at this point keywords are stored as identifiers. for (var i = 0; i < expandedTokens.Count; i++) { if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken) { var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax); expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind); } } var localExpandedTokens = expandedTokens; expandedTokens = expandedTokens .Select((x, i) => { var result = x .WithOriginalMacroReference(macroReference, i == 0) .WithSpan(macroReference.SourceRange, macroReference.Span); if (i == 0) { result = result.WithLeadingTrivia(token.LeadingTrivia); } if (i == localExpandedTokens.Count - 1) { result = result.WithTrailingTrivia(lastToken.TrailingTrivia); } return(result); }) .ToList(); _currentlyExpandingMacros.Pop(); return(true); }