private static void ParseModuleInnerAsync(ModuleInfo moduleInfo, BuildOptions moduleOptions, List <string> headerFiles, int workIndex)
{
// Find and load files with API tags
bool hasApi = false;
string headerFileContents = File.ReadAllText(headerFiles[workIndex]);
for (int j = 0; j < ApiTokens.SearchTags.Length; j++)
{
if (headerFileContents.Contains(ApiTokens.SearchTags[j]))
{
hasApi = true;
break;
}
}
if (!hasApi)
{
return;
}
// Process header file to generate the module API code reflection
var fileInfo = new FileInfo
{
Parent = null,
Name = headerFiles[workIndex],
Namespace = moduleInfo.Name,
};
lock (moduleInfo)
{
moduleInfo.AddChild(fileInfo);
}
try
{
// Tokenize the source
var tokenizer = new Tokenizer();
tokenizer.Tokenize(headerFileContents);
// Init the context
var context = new ParsingContext
{
File = fileInfo,
Tokenizer = tokenizer,
ScopeInfo = null,
CurrentAccessLevel = AccessLevel.Public,
ScopeTypeStack = new Stack <ApiTypeInfo>(),
ScopeAccessStack = new Stack <AccessLevel>(),
PreprocessorDefines = new Dictionary <string, string>(),
};
context.PreprocessorDefines.Add("FLAX_BUILD_BINDINGS", "1");
context.EnterScope(fileInfo);
// Process the source code
ApiTypeInfo scopeType = null;
Token prevToken = null;
while (true)
{
// Move to the next token
var token = tokenizer.NextToken();
if (token == null)
{
continue;
}
if (token.Type == TokenType.EndOfFile)
{
break;
}
// Parse API_.. tags in source code
if (token.Type == TokenType.Identifier && token.Value.StartsWith("API_", StringComparison.Ordinal))
{
if (string.Equals(token.Value, ApiTokens.Class, StringComparison.Ordinal))
{
if (!(context.ScopeInfo is FileInfo))
{
throw new NotImplementedException("TODO: add support for nested classes in scripting API");
}
var classInfo = ParseClass(ref context);
scopeType = classInfo;
context.ScopeInfo.AddChild(scopeType);
context.CurrentAccessLevel = AccessLevel.Public;
}
else if (string.Equals(token.Value, ApiTokens.Property, StringComparison.Ordinal))
{
var propertyInfo = ParseProperty(ref context);
}
else if (string.Equals(token.Value, ApiTokens.Function, StringComparison.Ordinal))
{
var functionInfo = ParseFunction(ref context);
if (context.ScopeInfo is ClassInfo classInfo)
{
classInfo.Functions.Add(functionInfo);
}
else if (context.ScopeInfo is StructureInfo structureInfo)
{
structureInfo.Functions.Add(functionInfo);
}
else if (context.ScopeInfo is InterfaceInfo interfaceInfo)
{
interfaceInfo.Functions.Add(functionInfo);
}
else
{
throw new Exception($"Not supported free-function {functionInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it.");
}
}
else if (string.Equals(token.Value, ApiTokens.Enum, StringComparison.Ordinal))
{
var enumInfo = ParseEnum(ref context);
context.ScopeInfo.AddChild(enumInfo);
}
else if (string.Equals(token.Value, ApiTokens.Struct, StringComparison.Ordinal))
{
var structureInfo = ParseStructure(ref context);
scopeType = structureInfo;
context.ScopeInfo.AddChild(scopeType);
context.CurrentAccessLevel = AccessLevel.Public;
}
else if (string.Equals(token.Value, ApiTokens.Field, StringComparison.Ordinal))
{
var fieldInfo = ParseField(ref context);
var scopeInfo = context.ValidScopeInfoFromStack;
if (scopeInfo is ClassInfo classInfo)
{
classInfo.Fields.Add(fieldInfo);
}
else if (scopeInfo is StructureInfo structureInfo)
{
structureInfo.Fields.Add(fieldInfo);
}
else
{
throw new Exception($"Not supported location for field {fieldInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class or structure to use API bindings for it.");
}
}
else if (string.Equals(token.Value, ApiTokens.Event, StringComparison.Ordinal))
{
var eventInfo = ParseEvent(ref context);
var scopeInfo = context.ValidScopeInfoFromStack;
if (scopeInfo is ClassInfo classInfo)
{
classInfo.Events.Add(eventInfo);
}
else
{
throw new Exception($"Not supported location for event {eventInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it.");
}
}
else if (string.Equals(token.Value, ApiTokens.Typedef, StringComparison.Ordinal))
{
var typeInfo = ParseTypedef(ref context);
fileInfo.AddChild(typeInfo);
}
else if (string.Equals(token.Value, ApiTokens.InjectCode, StringComparison.Ordinal))
{
var injectCodeInfo = ParseInjectCode(ref context);
fileInfo.AddChild(injectCodeInfo);
}
else if (string.Equals(token.Value, ApiTokens.Interface, StringComparison.Ordinal))
{
if (!(context.ScopeInfo is FileInfo))
{
throw new NotImplementedException("TODO: add support for nested interfaces in scripting API");
}
var interfaceInfo = ParseInterface(ref context);
scopeType = interfaceInfo;
context.ScopeInfo.AddChild(scopeType);
context.CurrentAccessLevel = AccessLevel.Public;
}
else if (string.Equals(token.Value, ApiTokens.AutoSerialization, StringComparison.Ordinal))
{
if (context.ScopeInfo is ClassInfo classInfo)
{
classInfo.IsAutoSerialization = true;
}
else if (context.ScopeInfo is StructureInfo structureInfo)
{
structureInfo.IsAutoSerialization = true;
}
else
{
throw new Exception($"Not supported location for {ApiTokens.AutoSerialization} at line {tokenizer.CurrentLine}. Place it in the class or structure that uses API bindings.");
}
}
}
// Track access level inside class
if (context.ScopeInfo != null && token.Type == TokenType.Colon && prevToken != null && prevToken.Type == TokenType.Identifier)
{
if (string.Equals(prevToken.Value, "public", StringComparison.Ordinal))
{
context.CurrentAccessLevel = AccessLevel.Public;
}
else if (string.Equals(prevToken.Value, "protected", StringComparison.Ordinal))
{
context.CurrentAccessLevel = AccessLevel.Protected;
}
else if (string.Equals(prevToken.Value, "private", StringComparison.Ordinal))
{
context.CurrentAccessLevel = AccessLevel.Private;
}
}
// Handle preprocessor blocks
if (token.Type == TokenType.Preprocessor)
{
token = tokenizer.NextToken();
switch (token.Value)
{
case "define":
{
token = tokenizer.NextToken();
var name = token.Value;
var value = string.Empty;
token = tokenizer.NextToken(true);
while (token.Type != TokenType.Newline)
{
value += token.Value;
token = tokenizer.NextToken(true);
}
value = value.Trim();
context.PreprocessorDefines[name] = value;
break;
}
case "if":
case "elif":
{
// Parse condition
var condition = string.Empty;
token = tokenizer.NextToken(true);
while (token.Type != TokenType.Newline)
{
var tokenValue = token.Value.Trim();
if (tokenValue.Length == 0)
{
token = tokenizer.NextToken(true);
continue;
}
// Very simple defines processing
var negate = tokenValue[0] == '!';
if (negate)
{
tokenValue = tokenValue.Substring(1);
}
tokenValue = ReplacePreProcessorDefines(tokenValue, context.PreprocessorDefines);
tokenValue = ReplacePreProcessorDefines(tokenValue, moduleOptions.PublicDefinitions);
tokenValue = ReplacePreProcessorDefines(tokenValue, moduleOptions.PrivateDefinitions);
tokenValue = ReplacePreProcessorDefines(tokenValue, moduleOptions.CompileEnv.PreprocessorDefinitions);
tokenValue = tokenValue.Replace("false", "0");
tokenValue = tokenValue.Replace("true", "1");
tokenValue = tokenValue.Replace("||", "|");
if (tokenValue.Length != 0 && tokenValue != "1" && tokenValue != "0" && tokenValue != "|")
{
tokenValue = "0";
}
if (negate)
{
tokenValue = tokenValue == "1" ? "0" : "1";
}
condition += tokenValue;
token = tokenizer.NextToken(true);
}
// Filter condition
bool modified;
do
{
modified = false;
if (condition.Contains("1|1"))
{
condition = condition.Replace("1|1", "1");
modified = true;
}
if (condition.Contains("1|0"))
{
condition = condition.Replace("1|0", "1");
modified = true;
}
if (condition.Contains("0|1"))
{
condition = condition.Replace("0|1", "1");
modified = true;
}
} while (modified);
// Skip chunk of code of condition fails
if (condition != "1")
{
ParsePreprocessorIf(fileInfo, tokenizer, ref token);
}
break;
}
case "ifdef":
{
// Parse condition
var define = string.Empty;
token = tokenizer.NextToken(true);
while (token.Type != TokenType.Newline)
{
define += token.Value;
token = tokenizer.NextToken(true);
}
// Check condition
define = define.Trim();
if (!context.PreprocessorDefines.ContainsKey(define) && !moduleOptions.CompileEnv.PreprocessorDefinitions.Contains(define))
{
ParsePreprocessorIf(fileInfo, tokenizer, ref token);
}
break;
}
}
}
// Scope tracking
if (token.Type == TokenType.LeftCurlyBrace)
{
context.ScopeTypeStack.Push(scopeType);
context.ScopeInfo = context.ScopeTypeStack.Peek();
scopeType = null;
}
else if (token.Type == TokenType.RightCurlyBrace)
{
context.ScopeTypeStack.Pop();
if (context.ScopeTypeStack.Count == 0)
{
throw new Exception($"Mismatch of the {{}} braces pair in file '{fileInfo.Name}' at line {tokenizer.CurrentLine}.");
}
context.ScopeInfo = context.ScopeTypeStack.Peek();
if (context.ScopeInfo is FileInfo)
{
context.CurrentAccessLevel = AccessLevel.Public;
}
}
prevToken = token;
}
}
catch (Exception ex)
{
Log.Error($"Failed to parse '{fileInfo.Name}' file to generate bindings.");
Log.Exception(ex);
throw;
}
}
private static ModuleInfo ParseModuleInner(BuildData buildData, Module module, BuildOptions moduleOptions = null)
{
if (buildData.ModulesInfo.TryGetValue(module, out var moduleInfo))
{
return(moduleInfo);
}
// Setup bindings module info descriptor
moduleInfo = new ModuleInfo
{
Module = module,
Name = module.BinaryModuleName,
Namespace = string.Empty,
Children = new List <ApiTypeInfo>(),
};
if (string.IsNullOrEmpty(moduleInfo.Name))
{
throw new Exception("Module name cannot be empty.");
}
buildData.ModulesInfo.Add(module, moduleInfo);
// Skip for modules that cannot have API bindings
if (module is ThirdPartyModule || !module.BuildNativeCode)
{
return(moduleInfo);
}
if (moduleOptions == null)
{
throw new Exception($"Cannot parse module {module.Name} without options.");
}
// Collect all header files that can have public symbols for API
var headerFiles = moduleOptions.SourceFiles.Where(x => x.EndsWith(".h")).ToList();
// Skip if no header files to process
if (headerFiles.Count == 0)
{
return(moduleInfo);
}
// Find and load files with API tags
string[] headerFilesContents = null;
//using (new ProfileEventScope("LoadHeaderFiles"))
{
var anyApi = false;
for (int i = 0; i < headerFiles.Count; i++)
{
// Skip scripting types definitions file
if (headerFiles[i].Replace('\\', '/').EndsWith("Engine/Core/Config.h", StringComparison.Ordinal) ||
headerFiles[i].EndsWith("EditorContextAPI.h", StringComparison.Ordinal))
{
continue;
}
// Check if file contains any valid API tag
var contents = File.ReadAllText(headerFiles[i]);
for (int j = 0; j < ApiTokens.SearchTags.Length; j++)
{
if (contents.Contains(ApiTokens.SearchTags[j]))
{
if (headerFilesContents == null)
{
headerFilesContents = new string[headerFiles.Count];
}
headerFilesContents[i] = contents;
anyApi = true;
break;
}
}
}
if (!anyApi)
{
return(moduleInfo);
}
}
// Skip if none of the headers was modified after last time generated C++ file was edited
// TODO: skip parsing if module has API cache file -> then load it from disk
/*if (!forceGenerate)
* {
* var lastGenerateTime = File.GetLastWriteTime(bindings.GeneratedCppFilePath);
* var anyModified = false;
* for (int i = 0; i < headerFiles.Count; i++)
* {
* if (File.GetLastWriteTime(headerFiles[i]) > lastGenerateTime)
* {
* anyModified = true;
* break;
* }
* }
*
* if (!anyModified)
* return;
* }*/
Log.Verbose($"Parsing API bindings for {module.Name} ({moduleInfo.Name})");
// Process all header files to generate the module API code reflection
var context = new ParsingContext
{
CurrentAccessLevel = AccessLevel.Public,
ScopeTypeStack = new Stack <ApiTypeInfo>(),
ScopeAccessStack = new Stack <AccessLevel>(),
PreprocessorDefines = new Dictionary <string, string>(),
};
for (int i = 0; i < headerFiles.Count; i++)
{
if (headerFilesContents[i] == null)
{
continue;
}
var fileInfo = new FileInfo
{
Parent = null,
Children = new List <ApiTypeInfo>(),
Name = headerFiles[i],
Namespace = moduleInfo.Name,
};
moduleInfo.AddChild(fileInfo);
try
{
// Tokenize the source
var tokenizer = new Tokenizer();
tokenizer.Tokenize(headerFilesContents[i]);
// Init the context
context.Tokenizer = tokenizer;
context.File = fileInfo;
context.ScopeInfo = null;
context.ScopeTypeStack.Clear();
context.ScopeAccessStack.Clear();
context.PreprocessorDefines.Clear();
context.EnterScope(fileInfo);
// Process the source code
ApiTypeInfo scopeType = null;
Token prevToken = null;
while (true)
{
// Move to the next token
var token = tokenizer.NextToken();
if (token == null)
{
continue;
}
if (token.Type == TokenType.EndOfFile)
{
break;
}
// Parse API_.. tags in source code
if (token.Type == TokenType.Identifier && token.Value.StartsWith("API_", StringComparison.Ordinal))
{
if (string.Equals(token.Value, ApiTokens.Class, StringComparison.Ordinal))
{
if (!(context.ScopeInfo is FileInfo))
{
throw new NotImplementedException("TODO: add support for nested classes in scripting API");
}
var classInfo = ParseClass(ref context);
scopeType = classInfo;
context.ScopeInfo.AddChild(scopeType);
context.CurrentAccessLevel = AccessLevel.Public;
}
else if (string.Equals(token.Value, ApiTokens.Property, StringComparison.Ordinal))
{
var propertyInfo = ParseProperty(ref context);
}
else if (string.Equals(token.Value, ApiTokens.Function, StringComparison.Ordinal))
{
var functionInfo = ParseFunction(ref context);
if (context.ScopeInfo is ClassInfo classInfo)
{
classInfo.Functions.Add(functionInfo);
}
else if (context.ScopeInfo is StructureInfo structureInfo)
{
structureInfo.Functions.Add(functionInfo);
}
else
{
throw new Exception($"Not supported free-function {functionInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it.");
}
}
else if (string.Equals(token.Value, ApiTokens.Enum, StringComparison.Ordinal))
{
var enumInfo = ParseEnum(ref context);
context.ScopeInfo.AddChild(enumInfo);
}
else if (string.Equals(token.Value, ApiTokens.Struct, StringComparison.Ordinal))
{
var structureInfo = ParseStructure(ref context);
scopeType = structureInfo;
context.ScopeInfo.AddChild(scopeType);
context.CurrentAccessLevel = AccessLevel.Public;
}
else if (string.Equals(token.Value, ApiTokens.Field, StringComparison.Ordinal))
{
var fieldInfo = ParseField(ref context);
var scopeInfo = context.ValidScopeInfoFromStack;
if (scopeInfo is ClassInfo classInfo)
{
classInfo.Fields.Add(fieldInfo);
}
else if (scopeInfo is StructureInfo structureInfo)
{
structureInfo.Fields.Add(fieldInfo);
}
else
{
throw new Exception($"Not supported location for field {fieldInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class or structure to use API bindings for it.");
}
}
else if (string.Equals(token.Value, ApiTokens.Event, StringComparison.Ordinal))
{
var eventInfo = ParseEvent(ref context);
var scopeInfo = context.ValidScopeInfoFromStack;
if (scopeInfo is ClassInfo classInfo)
{
classInfo.Events.Add(eventInfo);
}
else
{
throw new Exception($"Not supported location for event {eventInfo.Name} at line {tokenizer.CurrentLine}. Place it in the class to use API bindings for it.");
}
}
else if (string.Equals(token.Value, ApiTokens.InjectCppCode, StringComparison.Ordinal))
{
var injectCppCodeInfo = ParseInjectCppCode(ref context);
fileInfo.AddChild(injectCppCodeInfo);
}
else if (string.Equals(token.Value, ApiTokens.AutoSerialization, StringComparison.Ordinal))
{
if (context.ScopeInfo is ClassInfo classInfo)
{
classInfo.IsAutoSerialization = true;
}
else if (context.ScopeInfo is StructureInfo structureInfo)
{
structureInfo.IsAutoSerialization = true;
}
else
{
throw new Exception($"Not supported location for {ApiTokens.AutoSerialization} at line {tokenizer.CurrentLine}. Place it in the class or structure that uses API bindings.");
}
}
}
// Track access level inside class
if (context.ScopeInfo != null && token.Type == TokenType.Colon && prevToken != null && prevToken.Type == TokenType.Identifier)
{
if (string.Equals(prevToken.Value, "public", StringComparison.Ordinal))
{
context.CurrentAccessLevel = AccessLevel.Public;
}
else if (string.Equals(prevToken.Value, "protected", StringComparison.Ordinal))
{
context.CurrentAccessLevel = AccessLevel.Protected;
}
else if (string.Equals(prevToken.Value, "private", StringComparison.Ordinal))
{
context.CurrentAccessLevel = AccessLevel.Private;
}
}
// Handle preprocessor blocks
if (token.Type == TokenType.Preprocessor)
{
token = tokenizer.NextToken();
switch (token.Value)
{
case "define":
{
token = tokenizer.NextToken();
var name = token.Value;
var value = string.Empty;
token = tokenizer.NextToken(true);
while (token.Type != TokenType.Newline)
{
value += token.Value;
token = tokenizer.NextToken(true);
}
value = value.Trim();
context.PreprocessorDefines[name] = value;
break;
}
case "if":
{
// Parse condition
var condition = string.Empty;
token = tokenizer.NextToken(true);
while (token.Type != TokenType.Newline)
{
condition += token.Value;
token = tokenizer.NextToken(true);
}
// Replace contents with defines
condition = condition.Trim();
condition = ReplacePreProcessorDefines(condition, context.PreprocessorDefines.Keys);
condition = ReplacePreProcessorDefines(condition, moduleOptions.PublicDefinitions);
condition = ReplacePreProcessorDefines(condition, moduleOptions.PrivateDefinitions);
condition = ReplacePreProcessorDefines(condition, moduleOptions.CompileEnv.PreprocessorDefinitions);
condition = condition.Replace("false", "0");
condition = condition.Replace("true", "1");
// Check condition
// TODO: support expressions in preprocessor defines in API headers?
if (condition != "1")
{
// Skip chunk of code
ParsePreprocessorIf(fileInfo, tokenizer, ref token);
}
break;
}
case "ifdef":
{
// Parse condition
var define = string.Empty;
token = tokenizer.NextToken(true);
while (token.Type != TokenType.Newline)
{
define += token.Value;
token = tokenizer.NextToken(true);
}
// Check condition
define = define.Trim();
if (!context.PreprocessorDefines.ContainsKey(define) && !moduleOptions.CompileEnv.PreprocessorDefinitions.Contains(define))
{
ParsePreprocessorIf(fileInfo, tokenizer, ref token);
}
break;
}
}
}
// Scope tracking
if (token.Type == TokenType.LeftCurlyBrace)
{
context.ScopeTypeStack.Push(scopeType);
context.ScopeInfo = context.ScopeTypeStack.Peek();
scopeType = null;
}
else if (token.Type == TokenType.RightCurlyBrace)
{
context.ScopeTypeStack.Pop();
if (context.ScopeTypeStack.Count == 0)
{
throw new Exception($"Mismatch of the {{}} braces pair in file '{fileInfo.Name}' at line {tokenizer.CurrentLine}.");
}
context.ScopeInfo = context.ScopeTypeStack.Peek();
if (context.ScopeInfo is FileInfo)
{
context.CurrentAccessLevel = AccessLevel.Public;
}
}
prevToken = token;
}
}
catch (Exception ex)
{
Log.Error($"Failed to parse '{fileInfo.Name}' file to generate bindings.");
Log.Exception(ex);
throw;
}
}
// Initialize API
moduleInfo.Init(buildData);
return(moduleInfo);
}