private static ModuleInfo ParseModuleInner(BuildData buildData, Module module, BuildOptions moduleOptions = null) { // Setup bindings module info descriptor var moduleInfo = new ModuleInfo { Module = module, Name = module.BinaryModuleName, Namespace = string.Empty, }; if (string.IsNullOrEmpty(moduleInfo.Name)) { throw new Exception("Module name cannot be empty."); } buildData.ModulesInfo.Add(module, moduleInfo); // Skip for modules that cannot have API bindings if (!module.BuildCSharp || !module.BuildNativeCode) { return(moduleInfo); } if (moduleOptions == null) { throw new Exception($"Cannot parse module {module.Name} without options."); } // Collect all header files that can have public symbols for API var headerFiles = new List <string>(moduleOptions.SourceFiles.Count / 2); for (int i = 0; i < moduleOptions.SourceFiles.Count; i++) { if (moduleOptions.SourceFiles[i].EndsWith(".h", StringComparison.OrdinalIgnoreCase)) { headerFiles.Add(moduleOptions.SourceFiles[i]); } } if (headerFiles.Count == 0) { return(moduleInfo); } if (module.Name == "Core") { // Special case for Core module to ignore API tags defines for (int i = 0; i < headerFiles.Count; i++) { if (headerFiles[i].EndsWith("Config.h", StringComparison.Ordinal)) { headerFiles.RemoveAt(i); break; } } } // Sort file paths to have stable results headerFiles.Sort(); // Load cache using (new ProfileEventScope("LoadCache")) { if (LoadCache(ref moduleInfo, moduleOptions, headerFiles)) { buildData.ModulesInfo[module] = moduleInfo; // Initialize API using (new ProfileEventScope("Init")) { moduleInfo.Init(buildData); } return(moduleInfo); } } // Parse bindings Log.Verbose($"Parsing API bindings for {module.Name} ({moduleInfo.Name})"); int concurrency = Math.Min(Math.Max(1, (int)(Environment.ProcessorCount * Configuration.ConcurrencyProcessorScale)), Configuration.MaxConcurrency); concurrency = 1; // Disable concurrency for parsing (the gain is unnoticeable or even worse in some cases) if (concurrency == 1 || headerFiles.Count < 2 * concurrency) { // Single-threaded for (int i = 0; i < headerFiles.Count; i++) { using (new ProfileEventScope(Path.GetFileName(headerFiles[i]))) { ParseModuleInnerAsync(moduleInfo, moduleOptions, headerFiles, i); } } } else { // Multi-threaded ThreadPool.GetMinThreads(out var workerThreads, out var completionPortThreads); if (workerThreads != concurrency) { ThreadPool.SetMaxThreads(concurrency, completionPortThreads); } Parallel.For(0, headerFiles.Count, (i, state) => { using (new ProfileEventScope(Path.GetFileName(headerFiles[i]))) { ParseModuleInnerAsync(moduleInfo, moduleOptions, headerFiles, i); } }); } // Save cache using (new ProfileEventScope("SaveCache")) { try { SaveCache(moduleInfo, moduleOptions, headerFiles); } catch (Exception ex) { Log.Error($"Failed to save API cache for module {moduleInfo.Module.Name}"); Log.Exception(ex); } } // Initialize API using (new ProfileEventScope("Init")) { moduleInfo.Init(buildData); } return(moduleInfo); }
private static ModuleInfo ParseModuleInner(BuildData buildData, Module module, BuildOptions moduleOptions = null) { if (buildData.ModulesInfo.TryGetValue(module, out var moduleInfo)) { return(moduleInfo); } // Setup bindings module info descriptor moduleInfo = new ModuleInfo { Module = module, Name = module.BinaryModuleName, Namespace = string.Empty, Children = new List <ApiTypeInfo>(), }; if (string.IsNullOrEmpty(moduleInfo.Name)) { throw new Exception("Module name cannot be empty."); } buildData.ModulesInfo.Add(module, moduleInfo); // Skip for modules that cannot have API bindings if (module is ThirdPartyModule || !module.BuildNativeCode) { return(moduleInfo); } if (moduleOptions == null) { throw new Exception($"Cannot parse module {module.Name} without options."); } // Collect all header files that can have public symbols for API var headerFiles = moduleOptions.SourceFiles.Where(x => x.EndsWith(".h")).ToList(); // Skip if no header files to process if (headerFiles.Count == 0) { return(moduleInfo); } // Find and load files with API tags string[] headerFilesContents = null; //using (new ProfileEventScope("LoadHeaderFiles")) { var anyApi = false; for (int i = 0; i < headerFiles.Count; i++) { // Skip scripting types definitions file if (headerFiles[i].Replace('\\', '/').EndsWith("Engine/Core/Config.h", StringComparison.Ordinal) || headerFiles[i].EndsWith("EditorContextAPI.h", StringComparison.Ordinal)) { continue; } // Check if file contains any valid API tag var contents = File.ReadAllText(headerFiles[i]); for (int j = 0; j < ApiTokens.SearchTags.Length; j++) { if (contents.Contains(ApiTokens.SearchTags[j])) { if (headerFilesContents == null) { headerFilesContents = new string[headerFiles.Count]; } headerFilesContents[i] = contents; anyApi = true; break; } } } if (!anyApi) { return(moduleInfo); } } // Skip if none of the headers was modified after last time generated C++ file was edited // TODO: skip parsing if module has API cache file -> then load it from disk /*if (!forceGenerate) * { * var lastGenerateTime = File.GetLastWriteTime(bindings.GeneratedCppFilePath); * var anyModified = false; * for (int i = 0; i < headerFiles.Count; i++) * { * if (File.GetLastWriteTime(headerFiles[i]) > lastGenerateTime) * { * anyModified = true; * break; * } * } * * if (!anyModified) * return; * }*/ Log.Verbose($"Parsing API bindings for {module.Name} ({moduleInfo.Name})"); // Process all header files to generate the module API code reflection var context = new ParsingContext { CurrentAccessLevel = AccessLevel.Public, ScopeTypeStack = new Stack <ApiTypeInfo>(), ScopeAccessStack = new Stack <AccessLevel>(), PreprocessorDefines = new Dictionary <string, string>(), }; for (int i = 0; i < headerFiles.Count; i++) { if (headerFilesContents[i] == null) { continue; } var fileInfo = new FileInfo { Parent = null, Children = new List <ApiTypeInfo>(), Name = headerFiles[i], Namespace = moduleInfo.Name, }; moduleInfo.AddChild(fileInfo); try { // Tokenize the source var tokenizer = new Tokenizer(); tokenizer.Tokenize(headerFilesContents[i]); // Init the context context.Tokenizer = tokenizer; context.File = fileInfo; context.ScopeInfo = null; context.ScopeTypeStack.Clear(); context.ScopeAccessStack.Clear(); context.PreprocessorDefines.Clear(); context.EnterScope(fileInfo); // Process the source code ApiTypeInfo scopeType = null; Token prevToken = null; while (true) { // Move to the next token var token = tokenizer.NextToken(); if (token == null) { continue; } if (token.Type == TokenType.EndOfFile) { break; } // Parse API_.. tags in source code if (token.Type == TokenType.Identifier && token.Value.StartsWith("API_", StringComparison.Ordinal)) { if (string.Equals(token.Value, ApiTokens.Class, StringComparison.Ordinal)) { if (!(context.ScopeInfo is FileInfo)) { throw new NotImplementedException("TODO: add support for nested classes in scripting API"); } var classInfo = ParseClass(ref context); scopeType = classInfo; context.ScopeInfo.AddChild(scopeType); context.CurrentAccessLevel = AccessLevel.Public; } else if (string.Equals(token.Value, ApiTokens.Property, StringComparison.Ordinal)) { var propertyInfo = ParseProperty(ref context); } else if (string.Equals(token.Value, ApiTokens.Function, StringComparison.Ordinal)) { var functionInfo = ParseFunction(ref context); if (context.ScopeInfo is ClassInfo classInfo) { classInfo.Functions.Add(functionInfo); } else if (context.ScopeInfo is StructureInfo structureInfo) { structureInfo.Functions.Add(functionInfo); } else { throw new Exception($"Not supported free-function {functionInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it."); } } else if (string.Equals(token.Value, ApiTokens.Enum, StringComparison.Ordinal)) { var enumInfo = ParseEnum(ref context); context.ScopeInfo.AddChild(enumInfo); } else if (string.Equals(token.Value, ApiTokens.Struct, StringComparison.Ordinal)) { var structureInfo = ParseStructure(ref context); scopeType = structureInfo; context.ScopeInfo.AddChild(scopeType); context.CurrentAccessLevel = AccessLevel.Public; } else if (string.Equals(token.Value, ApiTokens.Field, StringComparison.Ordinal)) { var fieldInfo = ParseField(ref context); var scopeInfo = context.ValidScopeInfoFromStack; if (scopeInfo is ClassInfo classInfo) { classInfo.Fields.Add(fieldInfo); } else if (scopeInfo is StructureInfo structureInfo) { structureInfo.Fields.Add(fieldInfo); } else { throw new Exception($"Not supported location for field {fieldInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class or structure to use API bindings for it."); } } else if (string.Equals(token.Value, ApiTokens.Event, StringComparison.Ordinal)) { var eventInfo = ParseEvent(ref context); var scopeInfo = context.ValidScopeInfoFromStack; if (scopeInfo is ClassInfo classInfo) { classInfo.Events.Add(eventInfo); } else { throw new Exception($"Not supported location for event {eventInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it."); } } else if (string.Equals(token.Value, ApiTokens.InjectCppCode, StringComparison.Ordinal)) { var injectCppCodeInfo = ParseInjectCppCode(ref context); fileInfo.AddChild(injectCppCodeInfo); } else if (string.Equals(token.Value, ApiTokens.AutoSerialization, StringComparison.Ordinal)) { if (context.ScopeInfo is ClassInfo classInfo) { classInfo.IsAutoSerialization = true; } else if (context.ScopeInfo is StructureInfo structureInfo) { structureInfo.IsAutoSerialization = true; } else { throw new Exception($"Not supported location for {ApiTokens.AutoSerialization} at line {tokenizer.CurrentLine}. Place it in the class or structure that uses API bindings."); } } } // Track access level inside class if (context.ScopeInfo != null && token.Type == TokenType.Colon && prevToken != null && prevToken.Type == TokenType.Identifier) { if (string.Equals(prevToken.Value, "public", StringComparison.Ordinal)) { context.CurrentAccessLevel = AccessLevel.Public; } else if (string.Equals(prevToken.Value, "protected", StringComparison.Ordinal)) { context.CurrentAccessLevel = AccessLevel.Protected; } else if (string.Equals(prevToken.Value, "private", StringComparison.Ordinal)) { context.CurrentAccessLevel = AccessLevel.Private; } } // Handle preprocessor blocks if (token.Type == TokenType.Preprocessor) { token = tokenizer.NextToken(); switch (token.Value) { case "define": { token = tokenizer.NextToken(); var name = token.Value; var value = string.Empty; token = tokenizer.NextToken(true); while (token.Type != TokenType.Newline) { value += token.Value; token = tokenizer.NextToken(true); } value = value.Trim(); context.PreprocessorDefines[name] = value; break; } case "if": { // Parse condition var condition = string.Empty; token = tokenizer.NextToken(true); while (token.Type != TokenType.Newline) { condition += token.Value; token = tokenizer.NextToken(true); } // Replace contents with defines condition = condition.Trim(); condition = ReplacePreProcessorDefines(condition, context.PreprocessorDefines.Keys); condition = ReplacePreProcessorDefines(condition, moduleOptions.PublicDefinitions); condition = ReplacePreProcessorDefines(condition, moduleOptions.PrivateDefinitions); condition = ReplacePreProcessorDefines(condition, moduleOptions.CompileEnv.PreprocessorDefinitions); condition = condition.Replace("false", "0"); condition = condition.Replace("true", "1"); // Check condition // TODO: support expressions in preprocessor defines in API headers? if (condition != "1") { // Skip chunk of code ParsePreprocessorIf(fileInfo, tokenizer, ref token); } break; } case "ifdef": { // Parse condition var define = string.Empty; token = tokenizer.NextToken(true); while (token.Type != TokenType.Newline) { define += token.Value; token = tokenizer.NextToken(true); } // Check condition define = define.Trim(); if (!context.PreprocessorDefines.ContainsKey(define) && !moduleOptions.CompileEnv.PreprocessorDefinitions.Contains(define)) { ParsePreprocessorIf(fileInfo, tokenizer, ref token); } break; } } } // Scope tracking if (token.Type == TokenType.LeftCurlyBrace) { context.ScopeTypeStack.Push(scopeType); context.ScopeInfo = context.ScopeTypeStack.Peek(); scopeType = null; } else if (token.Type == TokenType.RightCurlyBrace) { context.ScopeTypeStack.Pop(); if (context.ScopeTypeStack.Count == 0) { throw new Exception($"Mismatch of the {{}} braces pair in file '{fileInfo.Name}' at line {tokenizer.CurrentLine}."); } context.ScopeInfo = context.ScopeTypeStack.Peek(); if (context.ScopeInfo is FileInfo) { context.CurrentAccessLevel = AccessLevel.Public; } } prevToken = token; } } catch (Exception ex) { Log.Error($"Failed to parse '{fileInfo.Name}' file to generate bindings."); Log.Exception(ex); throw; } } // Initialize API moduleInfo.Init(buildData); return(moduleInfo); }
private static void ParseModuleInnerAsync(ModuleInfo moduleInfo, BuildOptions moduleOptions, List <string> headerFiles, int workIndex) { // Find and load files with API tags bool hasApi = false; string headerFileContents = File.ReadAllText(headerFiles[workIndex]); for (int j = 0; j < ApiTokens.SearchTags.Length; j++) { if (headerFileContents.Contains(ApiTokens.SearchTags[j])) { hasApi = true; break; } } if (!hasApi) { return; } // Process header file to generate the module API code reflection var fileInfo = new FileInfo { Parent = null, Name = headerFiles[workIndex], Namespace = moduleInfo.Name, }; lock (moduleInfo) { moduleInfo.AddChild(fileInfo); } try { // Tokenize the source var tokenizer = new Tokenizer(); tokenizer.Tokenize(headerFileContents); // Init the context var context = new ParsingContext { File = fileInfo, Tokenizer = tokenizer, ScopeInfo = null, CurrentAccessLevel = AccessLevel.Public, ScopeTypeStack = new Stack <ApiTypeInfo>(), ScopeAccessStack = new Stack <AccessLevel>(), PreprocessorDefines = new Dictionary <string, string>(), }; context.EnterScope(fileInfo); // Process the source code ApiTypeInfo scopeType = null; Token prevToken = null; while (true) { // Move to the next token var token = tokenizer.NextToken(); if (token == null) { continue; } if (token.Type == TokenType.EndOfFile) { break; } // Parse API_.. tags in source code if (token.Type == TokenType.Identifier && token.Value.StartsWith("API_", StringComparison.Ordinal)) { if (string.Equals(token.Value, ApiTokens.Class, StringComparison.Ordinal)) { if (!(context.ScopeInfo is FileInfo)) { throw new NotImplementedException("TODO: add support for nested classes in scripting API"); } var classInfo = ParseClass(ref context); scopeType = classInfo; context.ScopeInfo.AddChild(scopeType); context.CurrentAccessLevel = AccessLevel.Public; } else if (string.Equals(token.Value, ApiTokens.Property, StringComparison.Ordinal)) { var propertyInfo = ParseProperty(ref context); } else if (string.Equals(token.Value, ApiTokens.Function, StringComparison.Ordinal)) { var functionInfo = ParseFunction(ref context); if (context.ScopeInfo is ClassInfo classInfo) { classInfo.Functions.Add(functionInfo); } else if (context.ScopeInfo is StructureInfo structureInfo) { structureInfo.Functions.Add(functionInfo); } else { throw new Exception($"Not supported free-function {functionInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it."); } } else if (string.Equals(token.Value, ApiTokens.Enum, StringComparison.Ordinal)) { var enumInfo = ParseEnum(ref context); context.ScopeInfo.AddChild(enumInfo); } else if (string.Equals(token.Value, ApiTokens.Struct, StringComparison.Ordinal)) { var structureInfo = ParseStructure(ref context); scopeType = structureInfo; context.ScopeInfo.AddChild(scopeType); context.CurrentAccessLevel = AccessLevel.Public; } else if (string.Equals(token.Value, ApiTokens.Field, StringComparison.Ordinal)) { var fieldInfo = ParseField(ref context); var scopeInfo = context.ValidScopeInfoFromStack; if (scopeInfo is ClassInfo classInfo) { classInfo.Fields.Add(fieldInfo); } else if (scopeInfo is StructureInfo structureInfo) { structureInfo.Fields.Add(fieldInfo); } else { throw new Exception($"Not supported location for field {fieldInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class or structure to use API bindings for it."); } } else if (string.Equals(token.Value, ApiTokens.Event, StringComparison.Ordinal)) { var eventInfo = ParseEvent(ref context); var scopeInfo = context.ValidScopeInfoFromStack; if (scopeInfo is ClassInfo classInfo) { classInfo.Events.Add(eventInfo); } else { throw new Exception($"Not supported location for event {eventInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it."); } } else if (string.Equals(token.Value, ApiTokens.InjectCppCode, StringComparison.Ordinal)) { var injectCppCodeInfo = ParseInjectCppCode(ref context); fileInfo.AddChild(injectCppCodeInfo); } else if (string.Equals(token.Value, ApiTokens.Interface, StringComparison.Ordinal)) { if (!(context.ScopeInfo is FileInfo)) { throw new NotImplementedException("TODO: add support for nested interfaces in scripting API"); } var interfaceInfo = ParseInterface(ref context); scopeType = interfaceInfo; context.ScopeInfo.AddChild(scopeType); context.CurrentAccessLevel = AccessLevel.Public; } else if (string.Equals(token.Value, ApiTokens.AutoSerialization, StringComparison.Ordinal)) { if (context.ScopeInfo is ClassInfo classInfo) { classInfo.IsAutoSerialization = true; } else if (context.ScopeInfo is StructureInfo structureInfo) { structureInfo.IsAutoSerialization = true; } else { throw new Exception($"Not supported location for {ApiTokens.AutoSerialization} at line {tokenizer.CurrentLine}. Place it in the class or structure that uses API bindings."); } } } // Track access level inside class if (context.ScopeInfo != null && token.Type == TokenType.Colon && prevToken != null && prevToken.Type == TokenType.Identifier) { if (string.Equals(prevToken.Value, "public", StringComparison.Ordinal)) { context.CurrentAccessLevel = AccessLevel.Public; } else if (string.Equals(prevToken.Value, "protected", StringComparison.Ordinal)) { context.CurrentAccessLevel = AccessLevel.Protected; } else if (string.Equals(prevToken.Value, "private", StringComparison.Ordinal)) { context.CurrentAccessLevel = AccessLevel.Private; } } // Handle preprocessor blocks if (token.Type == TokenType.Preprocessor) { token = tokenizer.NextToken(); switch (token.Value) { case "define": { token = tokenizer.NextToken(); var name = token.Value; var value = string.Empty; token = tokenizer.NextToken(true); while (token.Type != TokenType.Newline) { value += token.Value; token = tokenizer.NextToken(true); } value = value.Trim(); context.PreprocessorDefines[name] = value; break; } case "if": { // Parse condition var condition = string.Empty; token = tokenizer.NextToken(true); while (token.Type != TokenType.Newline) { var tokenValue = token.Value.Trim(); if (tokenValue.Length == 0) { token = tokenizer.NextToken(true); continue; } // Very simple defines processing tokenValue = ReplacePreProcessorDefines(tokenValue, context.PreprocessorDefines.Keys); tokenValue = ReplacePreProcessorDefines(tokenValue, moduleOptions.PublicDefinitions); tokenValue = ReplacePreProcessorDefines(tokenValue, moduleOptions.PrivateDefinitions); tokenValue = ReplacePreProcessorDefines(tokenValue, moduleOptions.CompileEnv.PreprocessorDefinitions); tokenValue = tokenValue.Replace("false", "0"); tokenValue = tokenValue.Replace("true", "1"); tokenValue = tokenValue.Replace("||", "|"); if (tokenValue.Length != 0 && tokenValue != "1" && tokenValue != "0" && tokenValue != "|") { tokenValue = "0"; } condition += tokenValue; token = tokenizer.NextToken(true); } // Filter condition condition = condition.Replace("1|1", "1"); condition = condition.Replace("1|0", "1"); condition = condition.Replace("0|1", "1"); // Skip chunk of code of condition fails if (condition != "1") { ParsePreprocessorIf(fileInfo, tokenizer, ref token); } break; } case "ifdef": { // Parse condition var define = string.Empty; token = tokenizer.NextToken(true); while (token.Type != TokenType.Newline) { define += token.Value; token = tokenizer.NextToken(true); } // Check condition define = define.Trim(); if (!context.PreprocessorDefines.ContainsKey(define) && !moduleOptions.CompileEnv.PreprocessorDefinitions.Contains(define)) { ParsePreprocessorIf(fileInfo, tokenizer, ref token); } break; } } } // Scope tracking if (token.Type == TokenType.LeftCurlyBrace) { context.ScopeTypeStack.Push(scopeType); context.ScopeInfo = context.ScopeTypeStack.Peek(); scopeType = null; } else if (token.Type == TokenType.RightCurlyBrace) { context.ScopeTypeStack.Pop(); if (context.ScopeTypeStack.Count == 0) { throw new Exception($"Mismatch of the {{}} braces pair in file '{fileInfo.Name}' at line {tokenizer.CurrentLine}."); } context.ScopeInfo = context.ScopeTypeStack.Peek(); if (context.ScopeInfo is FileInfo) { context.CurrentAccessLevel = AccessLevel.Public; } } prevToken = token; } } catch (Exception ex) { Log.Error($"Failed to parse '{fileInfo.Name}' file to generate bindings."); Log.Exception(ex); throw; } }
private static bool LoadCache(ref ModuleInfo moduleInfo, BuildOptions moduleOptions, List <string> headerFiles) { var path = GetCachePath(moduleInfo.Module, moduleOptions); if (!File.Exists(path)) { return(false); } try { using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var reader = new BinaryReader(stream, Encoding.UTF8)) { // Version var version = reader.ReadInt32(); if (version != CacheVersion) { return(false); } if (File.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks != reader.ReadInt64()) { return(false); } // Build options if (reader.ReadString() != moduleOptions.IntermediateFolder || reader.ReadInt32() != (int)moduleOptions.Platform.Target || reader.ReadInt32() != (int)moduleOptions.Architecture || reader.ReadInt32() != (int)moduleOptions.Configuration) { return(false); } var publicDefinitions = Read(reader, Utilities.GetEmptyArray <string>()); if (publicDefinitions.Length != moduleOptions.PublicDefinitions.Count || publicDefinitions.Any(x => !moduleOptions.PublicDefinitions.Contains(x))) { return(false); } var privateDefinitions = Read(reader, Utilities.GetEmptyArray <string>()); if (privateDefinitions.Length != moduleOptions.PrivateDefinitions.Count || privateDefinitions.Any(x => !moduleOptions.PrivateDefinitions.Contains(x))) { return(false); } var preprocessorDefinitions = Read(reader, Utilities.GetEmptyArray <string>()); if (preprocessorDefinitions.Length != moduleOptions.CompileEnv.PreprocessorDefinitions.Count || preprocessorDefinitions.Any(x => !moduleOptions.CompileEnv.PreprocessorDefinitions.Contains(x))) { return(false); } // Header files var headerFilesCount = reader.ReadInt32(); if (headerFilesCount != headerFiles.Count) { return(false); } for (int i = 0; i < headerFilesCount; i++) { var headerFile = headerFiles[i]; if (headerFile != reader.ReadString()) { return(false); } if (File.GetLastWriteTime(headerFile).Ticks > reader.ReadInt64()) { return(false); } } // Info var newModuleInfo = new ModuleInfo { Module = moduleInfo.Module, Name = moduleInfo.Name, Namespace = moduleInfo.Namespace, IsFromCache = true, }; newModuleInfo.Read(reader); // Skip parsing and use data loaded from cache moduleInfo = newModuleInfo; return(true); } } catch { // Skip loading cache return(false); } }