List <SimpleToken> TokenizeFileAndFillPreprocessorMacroCollection(string[] lines, PreprocessorMacroCollection collection) { List <SimpleToken> result = new List <SimpleToken>(); var tt = CppTokenizer.TokenType.Whitespace; var tokenizer = new CppTokenizer(""); PreprocessorMacroGroupBuilder builder = new PreprocessorMacroGroupBuilder(); for (int lineIndex = 0; lineIndex < lines.Length; lineIndex++) { var line = lines[lineIndex]; var tokens = tokenizer.TokenizeLine(line, ref tt, false); if (tokens.Length == 0) { continue; } if (tokens[0].Type == CppTokenizer.TokenType.PreprocessorDirective) { if (line.Substring(tokens[0].Start, tokens[0].Length) == "#define") { while (tokens.Last().Type == CppTokenizer.TokenType.Operator && line.Substring(tokens.Last().Start, tokens.Last().Length) == "\\") { //This is a multi-line #define statement. Combine it with the next line until we reach a line without a trailing backslash var newTokens = tokenizer.TokenizeLine(lines[++lineIndex], ref tt, false); tokens = tokens.Take(tokens.Length - 1).Concat(newTokens).ToArray(); } List <SimpleToken> macroTokens = new List <SimpleToken>(); List <string> comment = new List <string>(); foreach (var token in tokens.Skip(2)) { if (token.Type == CppTokenizer.TokenType.Comment) { comment.Add(line.Substring(token.Start, token.Length)); } else { macroTokens.Add(new SimpleToken(token.Type, line.Substring(token.Start, token.Length), lineIndex)); } } PreprocessorMacro macro = new PreprocessorMacro { Name = line.Substring(tokens[1].Start, tokens[1].Length), Value = macroTokens.ToArray(), CombinedComments = comment.Count == 0 ? null : string.Join(" ", comment.ToArray()) }; builder.OnPreprocessorMacroDefined(macro); collection.PreprocessorMacros[macro.Name] = macro; } } else { bool isFirstToken = true; builder.OnTokenizedLineProcessed(tokens, line); foreach (var token in tokens) { if (token.Type == CppTokenizer.TokenType.Comment && result.Count > 0 && result[result.Count - 1].Type == CppTokenizer.TokenType.Comment) { //Merge adjacent comments string separator = isFirstToken ? "\n" : " "; result[result.Count - 1] = result[result.Count - 1].WithAppendedText(separator + line.Substring(token.Start, token.Length)); } else { result.Add(new SimpleToken(token.Type, line.Substring(token.Start, token.Length), lineIndex)); } isFirstToken = false; } } } collection.PreprocessorMacroGroups = builder.ExportGroups(); return(result); }
public void OnPreprocessorMacroDefined(PreprocessorMacro macro) { _ConstructedGroup?.Macros?.Add(macro); }