public HlslLexer(SourceText text, ParserOptions options = null, IIncludeFileSystem includeFileSystem = null)
{
_includeFileResolver = new IncludeFileResolver(includeFileSystem ?? new DummyFileSystem());
_directives = DirectiveStack.Empty;
if (options != null)
{
foreach (var define in options.PreprocessorDefines)
{
var lexer = new HlslLexer(
SourceText.From($"#define {define.Key} {define.Value}",
"__ConfiguredPreprocessorDefinitions__.hlsl"));
lexer._mode = LexerMode.Directive;
lexer.ExpandMacros = false;
var dp = new DirectiveParser(lexer, _directives);
var directive = dp.ParseDirective(true, true, false);
_directives = directive.ApplyDirectives(_directives);
}
}
_options = options ?? new ParserOptions();
ExpandMacros = true;
_rootText = text;
FileSegments = new List <FileSegment>();
_includeStack = new Stack <IncludeContext>();
PushIncludeContext(text);
}
public static IReadOnlyList<SyntaxToken> ParseAllTokens(SourceText sourceText, IIncludeFileSystem fileSystem = null)
{
var tokens = new List<SyntaxToken>();
var lexer = new HlslLexer(sourceText, fileSystem: fileSystem);
SyntaxToken token;
do
{
tokens.Add(token = lexer.Lex(LexerMode.Syntax));
} while (token.Kind != SyntaxKind.EndOfFileToken);
return tokens;
}
private static SyntaxTree Parse(SourceText sourceText, ParserOptions options, IIncludeFileSystem fileSystem, Func<HlslParser, SyntaxNode> parseFunc)
{
var lexer = new HlslLexer(sourceText, options, fileSystem);
var parser = new HlslParser(lexer);
var result = new SyntaxTree(sourceText,
syntaxTree => new Tuple<SyntaxNode, List<FileSegment>>(
parseFunc(parser),
lexer.FileSegments));
Debug.WriteLine(DateTime.Now + " - Finished parsing");
return result;
}
public BaseMacroExpansionLexer(HlslLexer lexer)
{
_lexer = lexer;
}
private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List <SyntaxToken> expandedTokens)
{
expandedTokens = null;
// First, check if this token might be a macro.
DefineDirectiveTriviaSyntax directive;
if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined)
{
return(false);
}
// Check that this macro is not disabled.
if (_currentlyExpandingMacros.Contains(directive))
{
return(false);
}
MacroReference macroReference;
List <SyntaxToken> macroBody;
SyntaxToken lastToken;
switch (directive.Kind)
{
case SyntaxKind.FunctionLikeDefineDirectiveTrivia:
// For function-like macros, check for, and expand, macro arguments.
var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax)directive;
// ... check to see if the next token is an open paren.
if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken)
{
return(false);
}
// If it is, then parse the macro arguments, and
// check that we have the correct number of arguments.
ExpandMacros = false;
var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList();
ExpandMacros = true;
if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count)
{
expandedTokens = new List <SyntaxToken>
{
token
.WithDiagnostic(Diagnostic.Create(HlslMessageProvider.Instance, token.SourceRange, (int)DiagnosticId.NotEnoughMacroParameters, token.Text))
.WithTrailingTrivia(new[] { macroArguments })
};
return(true);
}
var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax)directive;
macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective);
// Expand arguments.
var expandedArguments = macroArguments.Arguments
.Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList())
.ToList();
// Replace parameters with possibly-expanded arguments.
macroBody = ReplaceParameters(
macroArguments.Arguments.ToList(), expandedArguments,
functionLikeDefineDirective.Parameters,
functionLikeDefineDirective.Body);
lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token;
break;
case SyntaxKind.ObjectLikeDefineDirectiveTrivia:
macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax)directive);
macroBody = directive.MacroBody;
lastToken = token;
break;
default:
throw new ArgumentOutOfRangeException();
}
// Push the current macro onto the stack - this prevents recursive expansion.
_currentlyExpandingMacros.Push(directive);
// Scan macro body for nested macros.
expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody));
// Relex identifier tokens, because at this point keywords are stored as identifiers.
for (var i = 0; i < expandedTokens.Count; i++)
{
if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken)
{
var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax);
expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind);
}
}
var localExpandedTokens = expandedTokens;
expandedTokens = expandedTokens
.Select((x, i) =>
{
var result = x
.WithOriginalMacroReference(macroReference, i == 0)
.WithSpan(macroReference.SourceRange, macroReference.Span);
if (i == 0)
{
result = result.WithLeadingTrivia(token.LeadingTrivia);
}
if (i == localExpandedTokens.Count - 1)
{
result = result.WithTrailingTrivia(lastToken.TrailingTrivia);
}
return(result);
})
.ToList();
_currentlyExpandingMacros.Pop();
return(true);
}
public DirectiveParser(HlslLexer lexer, DirectiveStack directiveStack)
: base(lexer, LexerMode.Directive)
{
_lexer = lexer;
_directiveStack = directiveStack;
}
public BaseMacroExpansionLexer(HlslLexer lexer)
{
_lexer = lexer;
}
private bool TryExpandMacro(SyntaxToken token, IMacroExpansionLexer lexer, out List<SyntaxToken> expandedTokens)
{
expandedTokens = null;
// First, check if this token might be a macro.
DefineDirectiveTriviaSyntax directive;
if (_directives.IsDefined(token.Text, out directive) != DefineState.Defined)
return false;
// Check that this macro is not disabled.
if (_currentlyExpandingMacros.Contains(directive))
return false;
MacroReference macroReference;
List<SyntaxToken> macroBody;
SyntaxToken lastToken;
switch (directive.Kind)
{
case SyntaxKind.FunctionLikeDefineDirectiveTrivia:
// For function-like macros, check for, and expand, macro arguments.
var functionLikeDefine = (FunctionLikeDefineDirectiveTriviaSyntax) directive;
// ... check to see if the next token is an open paren.
if (lexer.Peek(_mode).Kind != SyntaxKind.OpenParenToken)
return false;
// If it is, then parse the macro arguments, and
// check that we have the correct number of arguments.
ExpandMacros = false;
var macroArguments = new MacroArgumentsParser(lexer).ParseArgumentList();
ExpandMacros = true;
if (macroArguments.Arguments.Count != functionLikeDefine.Parameters.Parameters.Count)
{
expandedTokens = new List<SyntaxToken>
{
token
.WithDiagnostic(Diagnostic.Format(token.Span, DiagnosticId.NotEnoughMacroParameters, token.Text))
.WithTrailingTrivia(new[] { macroArguments })
};
return true;
}
var functionLikeDefineDirective = (FunctionLikeDefineDirectiveTriviaSyntax) directive;
macroReference = new FunctionLikeMacroReference(token, macroArguments, functionLikeDefineDirective);
// Expand arguments.
var expandedArguments = macroArguments.Arguments
.Select(x => ExpandNestedMacro(new NestedMacroExpansionLexer(x.Tokens)).ToList())
.ToList();
// Replace parameters with possibly-expanded arguments.
macroBody = ReplaceParameters(
macroArguments.Arguments.ToList(), expandedArguments,
functionLikeDefineDirective.Parameters,
functionLikeDefineDirective.Body);
lastToken = macroArguments.DescendantTokens().LastOrDefault(x => !x.IsMissing) ?? token;
break;
case SyntaxKind.ObjectLikeDefineDirectiveTrivia:
macroReference = new ObjectLikeMacroReference(token, (ObjectLikeDefineDirectiveTriviaSyntax) directive);
macroBody = directive.MacroBody;
lastToken = token;
break;
default:
throw new ArgumentOutOfRangeException();
}
// Push the current macro onto the stack - this prevents recursive expansion.
_currentlyExpandingMacros.Push(directive);
// Scan macro body for nested macros.
expandedTokens = ExpandNestedMacro(new NestedMacroExpansionLexer(macroBody));
// Relex identifier tokens, because at this point keywords are stored as identifiers.
for (var i = 0; i < expandedTokens.Count; i++)
if (expandedTokens[i].Kind == SyntaxKind.IdentifierToken)
{
var relexedToken = new HlslLexer(new StringText(expandedTokens[i].Text)).Lex(LexerMode.Syntax);
expandedTokens[i] = expandedTokens[i].WithKind(relexedToken.Kind).WithContextualKind(relexedToken.ContextualKind);
}
var localExpandedTokens = expandedTokens;
expandedTokens = expandedTokens
.Select((x, i) =>
{
var result = x
.WithOriginalMacroReference(macroReference, i == 0)
.WithSpan(macroReference.SourceRange, macroReference.Span);
if (i == 0)
result = result.WithLeadingTrivia(token.LeadingTrivia);
if (i == localExpandedTokens.Count - 1)
result = result.WithTrailingTrivia(lastToken.TrailingTrivia);
return result;
})
.ToList();
_currentlyExpandingMacros.Pop();
return true;
}
public DirectiveParser(HlslLexer lexer, DirectiveStack directiveStack)
: base(lexer, LexerMode.Directive)
{
_lexer = lexer;
_directiveStack = directiveStack;
}