private static void BuildNodes(CodeFileTokensBuilder builder, List <NavigationItem> navigation, List <CodeDiagnostic> diagnostic, CppAstNode root, string packageNamespace) { //Mapping of each namespace to it's leaf namespace nodes //These leaf nodes are processed to generate and group them together //C++ ast has declarations under same namespace in multiple files so these needs to be grouped for better presentation var namespaceLeafMap = new Dictionary <string, List <CppAstNode> >(); var types = new HashSet <string>(); var namespaceNodes = root.inner.Where(n => n.kind == NamespaceDeclKind && n.name == RootNamespace); bool foundFilterNamespace = false; foreach (var node in namespaceNodes) { var namespacebldr = new StringBuilder(); var leafNamespaceNode = node; var currentNode = node; //Iterate until leaf namespace node and generate full namespace while (currentNode?.kind == NamespaceDeclKind) { if (namespacebldr.Length > 0) { namespacebldr.Append("::"); } namespacebldr.Append(currentNode.name); leafNamespaceNode = currentNode; currentNode = currentNode.inner?.FirstOrDefault(n => n.kind == NamespaceDeclKind); } var nameSpace = namespacebldr.ToString(); // Skip <partialnamespace>::Details namespace if (nameSpace.EndsWith(DetailsNamespacePostfix)) { continue; } if (!foundFilterNamespace && nameSpace.StartsWith(packageNamespace)) { foundFilterNamespace = true; } if (!namespaceLeafMap.ContainsKey(nameSpace)) { namespaceLeafMap[nameSpace] = new List <CppAstNode>(); } namespaceLeafMap[nameSpace].Add(leafNamespaceNode); } foreach (var nameSpace in namespaceLeafMap.Keys) { // Filter namespace based on file name if any of the namespace matches file name pattern // If no namespace matches file name then allow all namespaces to be part of review to avoid mandating file name convention if (!foundFilterNamespace || nameSpace.StartsWith(packageNamespace)) { ProcessNamespaceNode(nameSpace); builder.NewLine(); builder.NewLine(); } } void ProcessNamespaceNode(string nameSpace) { NavigationItem currentNamespace = new NavigationItem() { NavigationId = nameSpace, Text = nameSpace, Tags = { { "TypeKind", "namespace" } } }; List <NavigationItem> currentNamespaceMembers = new List <NavigationItem>(); builder.Keyword("namespace"); builder.Space(); var namespaceTokens = nameSpace.Split("::"); foreach (var token in namespaceTokens) { builder.Text(token); builder.Space(); builder.Punctuation("{"); builder.Space(); } builder.NewLine(); //Process all nodes in namespace foreach (var leafNamespaceNode in namespaceLeafMap[nameSpace]) { if (leafNamespaceNode.inner != null) { foreach (var member in leafNamespaceNode.inner) { builder.IncrementIndent(); ProcessNode(member, currentNamespaceMembers, nameSpace); builder.DecrementIndent(); } } } currentNamespace.ChildItems = currentNamespaceMembers.ToArray(); navigation.Add(currentNamespace); builder.NewLine(); for (int i = 0; i < namespaceTokens.Length; i++) { builder.Punctuation("}"); } builder.NewLine(); } NavigationItem BuildDeclaration(string name, string kind, string parentId = "") { string definitionId = name; if (!string.IsNullOrEmpty(parentId)) { definitionId = parentId + "::" + name; } builder.Append(new CodeFileToken() { DefinitionId = definitionId, Kind = CodeFileTokenKind.TypeName, Value = name, }); types.Add(name); return(new NavigationItem() { NavigationId = definitionId, Text = name, Tags = { { "TypeKind", kind } } }); } void BuildMemberDeclaration(string containerName, string name, string id = "") { if (string.IsNullOrEmpty(id)) { id = name; } builder.Append(new CodeFileToken() { DefinitionId = containerName + "." + id, Kind = CodeFileTokenKind.MemberName, Value = name, }); } string GenerateUniqueMethodId(CppAstNode methodNode) { var bldr = new StringBuilder(); bldr.Append(methodNode.name); if (methodNode.inner != null) { foreach (var parameterNode in methodNode.inner) { if (parameterNode.kind == ParmVarDeclKind) { bldr.Append(":"); bldr.Append(parameterNode.type.Replace(" ", "_")); } } } bldr.Append("::"); var type = string.IsNullOrEmpty(methodNode.type) ? "void" : methodNode.type; bldr.Append(type.Replace(" ", "_")); return(bldr.ToString()); } NavigationItem ProcessClassNode(CppAstNode node, string parentName) { NavigationItem navigationItem = null; // Skip empty forward declarations if (node.inner == null) { return(navigationItem); } builder.Keyword(node.tagUsed); builder.Space(); if (!string.IsNullOrEmpty(node.name)) { navigationItem = BuildDeclaration(node.name, node.tagUsed, parentName); builder.Space(); } var memberNavigations = new List <NavigationItem>(); var parents = node.inner?.Where(n => parentTypes.Contains(n.kind)); if (parents != null) { bool first = true; //Show inheritance details foreach (var parent in parents) { if (first) { builder.Punctuation(":"); builder.Space(); first = false; } else { builder.Punctuation(","); builder.Space(); } builder.Keyword(parent.access); builder.Space(); if (parent.name != null) { BuildType(builder, parent.name, types); } } builder.Space(); } builder.NewLine(); builder.WriteIndent(); builder.Punctuation("{"); builder.NewLine(); //Double indentation for members since access modifier is not parent for members builder.IncrementIndent(); builder.IncrementIndent(); bool hasFoundDefaultAccessMembers = false; var id = parentName + "::" + node.name; if (node.inner != null) { bool isPrivateMember = false; string currentAccessModifier = ""; foreach (var childNode in node.inner) { if (childNode.kind == CxxRecordDeclKind && childNode.name == node.name) { continue; } // add public or protected access specifier if (childNode.kind == AccessSpecDeclKind) { //Skip all private members isPrivateMember = (childNode.access == AccessModifierPrivate); if (isPrivateMember) { continue; } currentAccessModifier = childNode.access; builder.DecrementIndent(); builder.WriteIndent(); builder.Keyword(childNode.access); builder.Punctuation(":"); builder.IncrementIndent(); builder.NewLine(); } else if (!isPrivateMember && !parentTypes.Contains(childNode.kind)) { if (string.IsNullOrEmpty(currentAccessModifier) && !hasFoundDefaultAccessMembers) { hasFoundDefaultAccessMembers = true; } ProcessNode(childNode, memberNavigations, id); } } } builder.DecrementIndent(); builder.DecrementIndent(); builder.WriteIndent(); builder.Punctuation("}"); builder.Punctuation(";"); builder.NewLine(); builder.NewLine(); if (navigationItem != null) { navigationItem.ChildItems = memberNavigations.ToArray(); } if (node.tagUsed == "class") { if (node.inner?.Any(n => n.isimplicit == true && n.kind == CxxConstructorDeclKind) == true) { diagnostic.Add(new CodeDiagnostic("", navigationItem.NavigationId, ImplicitConstrucorHintError, "")); } if (hasFoundDefaultAccessMembers) { diagnostic.Add(new CodeDiagnostic("", navigationItem.NavigationId, NonAccessModifierMemberError, "")); } } return(navigationItem); } NavigationItem ProcessEnumNode(CppAstNode node) { builder.Keyword("enum"); builder.Space(); var navigationItem = BuildDeclaration(node.name, "enum"); builder.NewLine(); builder.WriteIndent(); builder.Punctuation("{"); builder.NewLine(); builder.IncrementIndent(); if (node.inner != null) { foreach (var parameterNode in node.inner) { if (parameterNode.kind == EnumConstantDeclKind) { builder.WriteIndent(); BuildMemberDeclaration("", parameterNode.name); if (parameterNode.inner?.FirstOrDefault(n => n.kind == "ConstantExpr") is CppAstNode exprNode) { builder.Space(); builder.Punctuation("="); builder.Space(); BuildExpression(builder, exprNode); } builder.Punctuation(","); builder.NewLine(); } } } builder.DecrementIndent(); builder.WriteIndent(); builder.Punctuation("};"); builder.NewLine(); builder.NewLine(); return(navigationItem); } void ProcessFunctionDeclNode(CppAstNode node, string parentName) { if (node.isimplicit == true) { builder.Keyword("implicit"); builder.Space(); } if (node.isvirtual == true) { builder.Keyword("virtual"); builder.Space(); } if (node.inline == true) { builder.Keyword("inline"); builder.Space(); } if (!string.IsNullOrEmpty(node.storageClass)) { builder.Keyword(node.storageClass); builder.Space(); } if (node.type != null) { BuildType(builder, node.type, types); builder.Space(); } BuildMemberDeclaration(parentName, node.name, GenerateUniqueMethodId(node)); builder.Punctuation("("); bool first = true; if (node.inner != null) { bool isMultiLineArgs = node.inner.Count > 1; builder.IncrementIndent(); foreach (var parameterNode in node.inner) { if (parameterNode.kind == ParmVarDeclKind) { if (first) { first = false; } else { builder.Punctuation(","); builder.Space(); } if (isMultiLineArgs) { builder.NewLine(); builder.WriteIndent(); } BuildType(builder, parameterNode.type, types); if (!string.IsNullOrEmpty(parameterNode.name)) { builder.Space(); builder.Text(parameterNode.name); } } } builder.DecrementIndent(); } builder.Punctuation(")"); //Add any postfix keywords if signature has any. // Few expamples are 'const noexcept' if (!string.IsNullOrEmpty(node.keywords)) { foreach (var key in node.keywords.Split()) { builder.Space(); builder.Keyword(key); } } // If method is tagged as pure, delete, or default then it should be marked as "=<0|default|delete>" if (node.ispure == true) { builder.Space(); builder.Punctuation("="); builder.Space(); builder.Text("0"); } else if (node.isdefault == true) { builder.Space(); builder.Punctuation("="); builder.Space(); builder.Keyword("default"); } else if (node.isdelete == true) { builder.Space(); builder.Punctuation("="); builder.Space(); builder.Keyword("delete"); } builder.Punctuation(";"); builder.NewLine(); } void ProcessTemplateFuncDeclNode(CppAstNode node, string parentName) { builder.Keyword("template"); builder.Space(); if (node.inner != null) { bool first = true; builder.Punctuation("<"); foreach (var childnode in node.inner.Where(n => n.kind == TemplateTypeParmDeclKind)) { if (!first) { builder.Punctuation(","); builder.Space(); } builder.Text(childnode.name); } builder.Punctuation(">"); builder.Space(); var methodNode = node.inner.FirstOrDefault(node => node.kind == CxxMethodDeclKind); if (methodNode != null) { ProcessFunctionDeclNode(methodNode, parentName); } } } void ProcessTypeAlias(CppAstNode node) { builder.Keyword("using"); builder.Space(); builder.Text(node.name); builder.Space(); builder.Punctuation("="); builder.Space(); BuildType(builder, node.type, types); builder.Punctuation(";"); builder.NewLine(); } void ProcessVarDecNode(CppAstNode node, string parentName) { if (node.constexpr == true) { builder.Keyword("constexpr"); builder.Space(); } if (!string.IsNullOrEmpty(node.storageClass)) { builder.Keyword(node.storageClass); builder.Space(); } //Remove left most const from type if it is constexpr string type = node.type; if (node.constexpr == true && type.StartsWith("const")) { var regex = new Regex(Regex.Escape("const")); type = regex.Replace(type, "", 1).Trim(); } BuildType(builder, type, types); builder.Space(); BuildMemberDeclaration(parentName, node.name); if (node.inner?.FirstOrDefault() is CppAstNode exprNode) { builder.Space(); builder.Punctuation("="); builder.Space(); BuildExpression(builder, exprNode); } builder.Punctuation(";"); builder.NewLine(); } void ProcessNode(CppAstNode node, List <NavigationItem> navigationItems, string parentName) { NavigationItem currentNavItem = null; builder.WriteIndent(); switch (node.kind) { case CxxRecordDeclKind: { currentNavItem = ProcessClassNode(node, parentName); builder.NewLine(); break; } case CxxConstructorDeclKind: case CxxDestructorDeclKind: case FunctionDeclKind: case CxxMethodDeclKind: { ProcessFunctionDeclNode(node, parentName); break; } case EnumDeclKind: { currentNavItem = ProcessEnumNode(node); builder.NewLine(); break; } case FieldDeclKind: case VarDeclKind: { ProcessVarDecNode(node, parentName); break; } case TypeAliasDeclKind: { ProcessTypeAlias(node); break; } case FunctionTemplateDeclKind: { ProcessTemplateFuncDeclNode(node, parentName); break; } default: builder.Text(node.ToString()); builder.NewLine(); break; } if (currentNavItem != null && navigationItems != null) { navigationItems.Add(currentNavItem); } } void BuildType(CodeFileTokensBuilder builder, string type, HashSet <string> types) { foreach (Match typePartMatch in _typeTokenizer.Matches(type)) { var typePart = typePartMatch.ToString(); if (_keywords.Contains(typePart)) { builder.Keyword(typePart); } else if (typePart.Contains("::")) { // Handle type usage before it's defintition var typeNamespace = typePart.Substring(0, typePart.LastIndexOf("::")); string typeValue = typePart; string navigateToId = ""; if (types.Contains(typePart) || namespaceLeafMap.ContainsKey(typeNamespace)) { typeValue = typePart.Substring(typePart.LastIndexOf("::") + 2); navigateToId = typePart; } builder.Append(new CodeFileToken() { Kind = CodeFileTokenKind.TypeName, NavigateToId = navigateToId, Value = typeValue }); } else { builder.Text(typePart); } } } }
private static void BuildNodes(CodeFileTokensBuilder builder, List <NavigationItem> navigation, MemoryStream astStream) { Span <byte> ast = astStream.ToArray(); while (ast.Length > 2) { Utf8JsonReader reader = new Utf8JsonReader(ast); var astNode = JsonSerializer.Deserialize <CAstNode>(ref reader); ast = ast.Slice((int)reader.BytesConsumed); var queue = new Queue <CAstNode>(astNode.inner); var types = new HashSet <string>(); foreach (var node in queue) { if (node.kind == "TypedefDecl") { types.Add(node.name); } } NavigationItem currentFileItem = null; List <NavigationItem> currentFileMembers = new List <NavigationItem>(); while (queue.TryDequeue(out var node)) { if (node.isImplicit == true || node.loc?.includedFrom?.file != null || node.loc?.spellingLoc?.includedFrom?.file != null) { continue; } var file = node.loc.file; if (file != null && currentFileItem == null) { currentFileItem = new NavigationItem() { NavigationId = file, Text = file, Tags = { { "TypeKind", "namespace" } } }; builder.Append(new CodeFileToken() { DefinitionId = file, Value = "// " + file, Kind = CodeFileTokenKind.Comment, }); builder.NewLine(); builder.Space(); builder.NewLine(); } bool TryDequeTypeDef(out CAstNode typedefNode) { if (queue.TryPeek(out typedefNode)) { if (typedefNode.kind == "TypedefDecl") { queue.Dequeue(); return(true); } } typedefNode = null; return(false); } void BuildDeclaration(string name, string kind) { builder.Append(new CodeFileToken() { DefinitionId = name, Kind = CodeFileTokenKind.TypeName, Value = name, }); currentFileMembers.Add(new NavigationItem() { NavigationId = name, Text = name, Tags = { { "TypeKind", kind } } }); } void BuildMemberDeclaration(string containerName, string name) { builder.Append(new CodeFileToken() { DefinitionId = containerName + "." + name, Kind = CodeFileTokenKind.MemberName, Value = name, }); } switch (node.kind) { case "FunctionDecl": { var type = node.type.qualType; var returnType = type.Split(" ")[0]; BuildType(builder, returnType, types); builder.Space(); BuildDeclaration(node.name, "method"); builder.Punctuation("("); builder.IncrementIndent(); bool first = true; foreach (var parameterNode in node.inner) { if (parameterNode.kind == "ParmVarDecl") { if (first) { builder.NewLine(); first = false; } builder.WriteIndent(); BuildType(builder, parameterNode.type.qualType, types); builder.Space(); builder.Text(parameterNode.name); builder.Punctuation(","); builder.NewLine(); } } builder.DecrementIndent(); builder.Punctuation(");"); builder.NewLine(); break; } case "EnumDecl": { if (TryDequeTypeDef(out var typeDef)) { builder.Keyword("typedef"); builder.Space(); } builder.Keyword("enum"); builder.NewLine(); builder.Punctuation("{"); builder.NewLine(); builder.IncrementIndent(); foreach (var parameterNode in node.inner) { if (parameterNode.kind == "EnumConstantDecl") { builder.WriteIndent(); BuildMemberDeclaration(typeDef?.name, parameterNode.name); if (parameterNode.inner?.FirstOrDefault(n => n.kind == "ConstantExpr") is CAstNode exprNode) { builder.Space(); builder.Punctuation("="); builder.Space(); BuildExpression(builder, exprNode); } builder.Punctuation(","); builder.NewLine(); } } builder.DecrementIndent(); builder.Punctuation("}"); if (typeDef != null) { builder.Space(); BuildDeclaration(typeDef.name, "enum"); } builder.Punctuation(";"); builder.NewLine(); break; } case "TypedefDecl": { builder.Keyword("typedef "); foreach (var typeDefValueNode in node.inner) { var type = typeDefValueNode.type?.qualType; if (type != null) { BuildType(builder, type, types); } } builder.Space(); BuildDeclaration(node.name, "class"); builder.Punctuation(";"); builder.NewLine(); break; } case "VarDecl": { BuildType(builder, node.type.qualType, types); builder.Space(); BuildDeclaration(node.name, "unknown"); builder.Punctuation(";"); builder.NewLine(); break; } case "RecordDecl": { if (TryDequeTypeDef(out var typeDef)) { builder.Keyword("typedef"); builder.Space(); } builder.Keyword("struct"); builder.NewLine(); builder.Punctuation("{"); builder.NewLine(); builder.IncrementIndent(); if (node.inner != null) { foreach (var parameterNode in node.inner) { if (parameterNode.kind == "FieldDecl") { builder.WriteIndent(); BuildType(builder, parameterNode.type.qualType, types); builder.Space(); BuildMemberDeclaration(typeDef?.name, parameterNode.name); builder.Punctuation(","); builder.NewLine(); } } } builder.DecrementIndent(); builder.Punctuation("}"); if (typeDef != null) { builder.Space(); BuildDeclaration(typeDef.name, "struct"); } builder.Punctuation(";"); builder.NewLine(); break; } default: builder.Text(node.ToString()); break; } builder.Space(); builder.NewLine(); } if (currentFileItem != null) { currentFileItem.ChildItems = currentFileMembers.ToArray(); navigation.Add(currentFileItem); } } }