private List <SyntaxToken> ExpandNestedMacro(NestedMacroExpansionLexer lexer)
{
var result = new List <SyntaxToken>();
SyntaxToken token;
while ((token = lexer.GetNextToken()) != null)
{
// Do token pasting (##).
if (lexer.Peek(0) != null && lexer.Peek(0).Kind == SyntaxKind.HashHashToken &&
lexer.Peek(1) != null && lexer.Peek(1).Kind != SyntaxKind.HashHashToken)
{
lexer.GetNextToken();
var concatenatedText = token.Text + lexer.GetNextToken().Text;
token = new HlslLexer(new StringText(concatenatedText)).Lex(LexerMode.Syntax);
}
List <SyntaxToken> expandedTokens;
if (TryExpandMacro(token, lexer, out expandedTokens))
{
result.AddRange(expandedTokens);
}
else
{
result.Add(token);
}
}
return(result);
}
public static IReadOnlyList<SyntaxToken> ParseAllTokens(SourceText sourceText, IIncludeFileSystem fileSystem = null)
{
var tokens = new List<SyntaxToken>();
var lexer = new HlslLexer(sourceText, fileSystem: fileSystem);
SyntaxToken token;
do
{
tokens.Add(token = lexer.Lex(LexerMode.Syntax));
} while (token.Kind != SyntaxKind.EndOfFileToken);
return tokens;
}
private static SyntaxTree Parse(SourceText sourceText, ParserOptions options, IIncludeFileSystem fileSystem, Func<HlslParser, SyntaxNode> parseFunc)
{
var lexer = new HlslLexer(sourceText, options, fileSystem);
var parser = new HlslParser(lexer);
var result = new SyntaxTree(sourceText,
syntaxTree => new Tuple<SyntaxNode, List<FileSegment>>(
parseFunc(parser),
lexer.FileSegments));
Debug.WriteLine(DateTime.Now + " - Finished parsing");
return result;
}
public DirectiveParser(HlslLexer lexer, DirectiveStack directiveStack)
: base(lexer, LexerMode.Directive)
{
_lexer = lexer;
_directiveStack = directiveStack;
}
public DirectiveParser(HlslLexer lexer, DirectiveStack directiveStack)
: base(lexer, LexerMode.Directive)
{
_lexer = lexer;
_directiveStack = directiveStack;
}
public BaseMacroExpansionLexer(HlslLexer lexer)
{
_lexer = lexer;
}
public BaseMacroExpansionLexer(HlslLexer lexer)
{
_lexer = lexer;
}
private List<SyntaxToken> ExpandNestedMacro(NestedMacroExpansionLexer lexer)
{
var result = new List<SyntaxToken>();
SyntaxToken token;
while ((token = lexer.GetNextToken()) != null)
{
// Do token pasting (##).
if (lexer.Peek(0) != null && lexer.Peek(0).Kind == SyntaxKind.HashHashToken
&& lexer.Peek(1) != null && lexer.Peek(1).Kind != SyntaxKind.HashHashToken)
{
lexer.GetNextToken();
var concatenatedText = token.Text + lexer.GetNextToken().Text;
token = new HlslLexer(new StringText(concatenatedText)).Lex(LexerMode.Syntax);
}
List<SyntaxToken> expandedTokens;
if (TryExpandMacro(token, lexer, out expandedTokens))
result.AddRange(expandedTokens);
else
result.Add(token);
}
return result;
}
private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List<SyntaxToken> expandedTokens)
{
expandedTokens = null;
// First, check if this token might be a macro.
DefineDirectiveTriviaSyntax directive;
if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined)
return false;
// Check that this macro is not disabled.
if (_currentlyExpandingMacros.Contains(directive))
return false;
MacroReference macroReference;
List<SyntaxToken> macroBody;
SyntaxToken lastToken;
switch (directive.Kind)
{
case SyntaxKind.FunctionLikeDefineDirectiveTrivia:
// For function-like macros, check for, and expand, macro arguments.
var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax) directive;
// ... check to see if the next token is an open paren.
if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken)
return false;
// If it is, then parse the macro arguments, and
// check that we have the correct number of arguments.
ExpandMacros = false;
var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList();
ExpandMacros = true;
if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count)
{
expandedTokens = new List<SyntaxToken>
{
token
.WithDiagnostic(Diagnostic.Format(token.Span, DiagnosticId.NotEnoughMacroParameters, token.Text))
.WithTrailingTrivia(new[] { macroArguments })
};
return true;
}
var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax) directive;
macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective);
// Expand arguments.
var expandedArguments = macroArguments.Arguments
.Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList())
.ToList();
// Replace parameters with possibly-expanded arguments.
macroBody = ReplaceParameters(
macroArguments.Arguments.ToList(), expandedArguments,
functionLikeDefineDirective.Parameters,
functionLikeDefineDirective.Body);
lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token;
break;
case SyntaxKind.ObjectLikeDefineDirectiveTrivia:
macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax) directive);
macroBody = directive.MacroBody;
lastToken = token;
break;
default:
throw new ArgumentOutOfRangeException();
}
// Push the current macro onto the stack - this prevents recursive expansion.
_currentlyExpandingMacros.Push(directive);
// Scan macro body for nested macros.
expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody));
// Relex identifier tokens, because at this point keywords are stored as identifiers.
for (var i = 0; i < expandedTokens.Count; i++)
if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken)
{
var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax);
expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind);
}
var localExpandedTokens = expandedTokens;
expandedTokens = expandedTokens
.Select((x, i) =>
{
var result = x
.WithOriginalMacroReference(macroReference, i == 0)
.WithSpan(macroReference.SourceRange, macroReference.Span);
if (i == 0)
result = result.WithLeadingTrivia(token.LeadingTrivia);
if (i == localExpandedTokens.Count - 1)
result = result.WithTrailingTrivia(lastToken.TrailingTrivia);
return result;
})
.ToList();
_currentlyExpandingMacros.Pop();
return true;
}
private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List <SyntaxToken> expandedTokens)
{
expandedTokens = null;
// First, check if this token might be a macro.
DefineDirectiveTriviaSyntax directive;
if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined)
{
return(false);
}
// Check that this macro is not disabled.
if (_currentlyExpandingMacros.Contains(directive))
{
return(false);
}
MacroReference macroReference;
List <SyntaxToken> macroBody;
SyntaxToken lastToken;
switch (directive.Kind)
{
case SyntaxKind.FunctionLikeDefineDirectiveTrivia:
// For function-like macros, check for, and expand, macro arguments.
var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax)directive;
// ... check to see if the next token is an open paren.
if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken)
{
return(false);
}
// If it is, then parse the macro arguments, and
// check that we have the correct number of arguments.
ExpandMacros = false;
var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList();
ExpandMacros = true;
if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count)
{
expandedTokens = new List <SyntaxToken>
{
token
.WithDiagnostic(Diagnostic.Format(token.Span, DiagnosticId.NotEnoughMacroParameters, token.Text))
.WithTrailingTrivia(new[] { macroArguments })
};
return(true);
}
var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax)directive;
macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective);
// Expand arguments.
var expandedArguments = macroArguments.Arguments
.Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList())
.ToList();
// Replace parameters with possibly-expanded arguments.
macroBody = ReplaceParameters(
macroArguments.Arguments.ToList(), expandedArguments,
functionLikeDefineDirective.Parameters,
functionLikeDefineDirective.Body);
lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token;
break;
case SyntaxKind.ObjectLikeDefineDirectiveTrivia:
macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax)directive);
macroBody = directive.MacroBody;
lastToken = token;
break;
default:
throw new ArgumentOutOfRangeException();
}
// Push the current macro onto the stack - this prevents recursive expansion.
_currentlyExpandingMacros.Push(directive);
// Scan macro body for nested macros.
expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody));
// Relex identifier tokens, because at this point keywords are stored as identifiers.
for (var i = 0; i < expandedTokens.Count; i++)
{
if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken)
{
var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax);
expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind);
}
}
var localExpandedTokens = expandedTokens;
expandedTokens = expandedTokens
.Select((x, i) =>
{
var result = x
.WithOriginalMacroReference(macroReference, i == 0)
.WithSpan(macroReference.SourceRange, macroReference.Span);
if (i == 0)
{
result = result.WithLeadingTrivia(token.LeadingTrivia);
}
if (i == localExpandedTokens.Count - 1)
{
result = result.WithTrailingTrivia(lastToken.TrailingTrivia);
}
return(result);
})
.ToList();
_currentlyExpandingMacros.Pop();
return(true);
}