List <SimpleToken> TokenizeFileAndFillPreprocessorMacroCollection(string[] lines, PreprocessorMacroCollection collection)
{
List <SimpleToken> result = new List <SimpleToken>();
var tt = CppTokenizer.TokenType.Whitespace;
var tokenizer = new CppTokenizer("");
PreprocessorMacroGroupBuilder builder = new PreprocessorMacroGroupBuilder();
for (int lineIndex = 0; lineIndex < lines.Length; lineIndex++)
{
var line = lines[lineIndex];
var tokens = tokenizer.TokenizeLine(line, ref tt, false);
if (tokens.Length == 0)
{
continue;
}
if (tokens[0].Type == CppTokenizer.TokenType.PreprocessorDirective)
{
if (line.Substring(tokens[0].Start, tokens[0].Length) == "#define")
{
while (tokens.Last().Type == CppTokenizer.TokenType.Operator &&
line.Substring(tokens.Last().Start, tokens.Last().Length) == "\\")
{
//This is a multi-line #define statement. Combine it with the next line until we reach a line without a trailing backslash
var newTokens = tokenizer.TokenizeLine(lines[++lineIndex], ref tt, false);
tokens = tokens.Take(tokens.Length - 1).Concat(newTokens).ToArray();
}
List <SimpleToken> macroTokens = new List <SimpleToken>();
List <string> comment = new List <string>();
foreach (var token in tokens.Skip(2))
{
if (token.Type == CppTokenizer.TokenType.Comment)
{
comment.Add(line.Substring(token.Start, token.Length));
}
else
{
macroTokens.Add(new SimpleToken(token.Type, line.Substring(token.Start, token.Length), lineIndex));
}
}
PreprocessorMacro macro = new PreprocessorMacro
{
Name = line.Substring(tokens[1].Start, tokens[1].Length),
Value = macroTokens.ToArray(),
CombinedComments = comment.Count == 0 ? null : string.Join(" ", comment.ToArray())
};
builder.OnPreprocessorMacroDefined(macro);
collection.PreprocessorMacros[macro.Name] = macro;
}
}
else
{
bool isFirstToken = true;
builder.OnTokenizedLineProcessed(tokens, line);
foreach (var token in tokens)
{
if (token.Type == CppTokenizer.TokenType.Comment && result.Count > 0 && result[result.Count - 1].Type == CppTokenizer.TokenType.Comment)
{
//Merge adjacent comments
string separator = isFirstToken ? "\n" : " ";
result[result.Count - 1] = result[result.Count - 1].WithAppendedText(separator + line.Substring(token.Start, token.Length));
}
else
{
result.Add(new SimpleToken(token.Type, line.Substring(token.Start, token.Length), lineIndex));
}
isFirstToken = false;
}
}
}
collection.PreprocessorMacroGroups = builder.ExportGroups();
return(result);
}
public void OnPreprocessorMacroDefined(PreprocessorMacro macro)
{
_ConstructedGroup?.Macros?.Add(macro);
}
