private TokenizeResult TokenizeCpp(string text, TextPosition pos, int length, bool allowDoxy)
{
TokenizeResult result = new TokenizeResult();
Stopwatch timer = new Stopwatch();
timer.Restart();
List <CppToken> cppTokens = new List <CppToken>();
using (CppLexer cppLexer = new CppLexer(text, pos, length))
{
cppTokens.AddRange(cppLexer.Tokenize());
result.AddErrors(cppLexer.LexErrors);
}
timer.Stop();
result.Stats.CppDuration += timer.Elapsed;
foreach (CppToken token in cppTokens)
{
if (allowDoxy && (token.Kind == CppTokenKind.MultiLineCommentDoc || token.Kind == CppTokenKind.SingleLineCommentDoc))
{
result.AddToken(token);
using (TokenizeResult doxyRes = TokenizeDoxy(text, token.Position, token.Length))
{
result.Stats.DoxyDuration += doxyRes.Stats.DoxyDuration;
result.Stats.HtmlDuration += doxyRes.Stats.HtmlDuration;
result.AddTokens(doxyRes.Tokens);
result.AddErrors(doxyRes.Errors);
}
}
else
{
result.AddToken(token);
}
}
return(result);
}
const string ErrMsgFormat = "-- line {0} col {1}: {2}"; // 0=line, 1=column, 2=text
public CppParser(CppLexer lexer)
{
this.errors = lexer.Errors;
errors.SynErr = new ErrorCodeProc(SynErr);
this.lexer = (CppLexer)lexer;
this.stack = new Stack <AstNode>();
}
public void Dispose()
{
errors = null;
if (lexer != null)
{
lexer.Dispose();
}
lexer = null;
}
private ColourisedQuickInfoContent CreateColourisedContent(string content)
{
// TODO Investigate using C++'s colouriser instead of tokenising the content
// ourselves. We might then be able to colour types and identifiers.
var lexer = new CppLexer();
CppToken[] tokens = lexer.GetTokens(content);
return(new ColourisedQuickInfoContent(tokens));
}
private CppParser BuildParser(string sourceCode)
{
// Parse the file
var inputStream = new AntlrInputStream(sourceCode);
var lexer = new CppLexer(inputStream);
var tokenStream = new CommonTokenStream(lexer);
var parser = new CppParser(tokenStream);
// Setup error handling
lexer.RemoveErrorListeners();
lexer.AddErrorListener(new LexerExceptionErrorListener());
parser.RemoveErrorListeners();
parser.AddErrorListener(new ParserExceptionErrorListener());
return(parser);
}
private void RunTest(string sourceCode, int expectedToken)
{
// Parse the file
var inputStream = new AntlrInputStream(sourceCode);
var lexer = new CppLexer(inputStream);
// Setup error handling
lexer.RemoveErrorListeners();
lexer.AddErrorListener(new LexerExceptionErrorListener());
// Match all tokens
var tokens = lexer.GetAllTokens();
// Verify we got the single expected token
Assert.Equal(1, tokens.Count);
Assert.Equal(expectedToken, tokens[0].Type);
}
public static TranslationUnit GenerateAST(AntlrInputStream inputStream)
{
// Parse the file
var lexer = new CppLexer(inputStream);
var tokenStream = new CommonTokenStream(lexer);
var parser = new CppParser(tokenStream);
// var tokens = lexer.GetAllTokens().Select(token => lexer.Vocabulary.GetSymbolicName(token.Type)).ToList();
// Setup error handling
lexer.RemoveErrorListeners();
lexer.AddErrorListener(new LexerExceptionErrorListener());
parser.RemoveErrorListeners();
parser.AddErrorListener(new ParserExceptionErrorListener());
// Parse the concrete syntax tree
var translationUnit = parser.translationUnit();
// Convert the the abstract syntax tree
var visitor = new ASTVisitor();
var ast = (TranslationUnit)visitor.Visit(translationUnit);
return(ast);
}
static int Main(string[] args)
{
if (args.Length < 1)
{
Console.Error.WriteLine("Missing filepath argument!");
return(-1);
}
string filePath = args[0];
string source = File.ReadAllText(filePath);
TokenizerType[] tokenizerTypes = new[] { TokenizerType.Superpower, TokenizerType.DoxygenCpp, TokenizerType.CustomCpp };
const int numberOfIterationsPerType = 1;
Dictionary <TokenizerType, List <TimeSpan> > durations = new Dictionary <TokenizerType, List <TimeSpan> >();
Dictionary <TokenizerType, List <CToken> > tokenMap = new Dictionary <TokenizerType, List <CToken> >();
foreach (TokenizerType tokenizerType in tokenizerTypes)
{
List <TimeSpan> spans = new List <TimeSpan>();
durations.Add(tokenizerType, spans);
List <CToken> outTokens = new List <CToken>();
tokenMap.Add(tokenizerType, outTokens);
for (int iteration = 1; iteration <= numberOfIterationsPerType; ++iteration)
{
outTokens.Clear();
Console.WriteLine($"{tokenizerType} tokenizer[{iteration}/{numberOfIterationsPerType}] start...");
Stopwatch timer = Stopwatch.StartNew();
switch (tokenizerType)
{
case TokenizerType.Superpower:
{
var tokenizer = new CSuperPowerTokenizer();
var tokens = tokenizer.Tokenize(source);
foreach (var token in tokens)
{
if (token.Kind == CppTokenKind.Eof)
{
break;
}
var start = new TextPosition(token.Position.Absolute, token.Position.Line - 1, token.Position.Column - 1);
var end = new TextPosition(token.Position.Absolute + token.Span.Length, token.Span.Position.Line - 1, token.Span.Position.Column - 1);
var value = source.Substring(start.Index, end.Index - start.Index);
outTokens.Add(new CToken(token.Kind, start, end, value));
}
}
break;
case TokenizerType.CustomCpp:
{
using (var stream = new BasicTextStream(source, new TextPosition(0), source.Length))
{
CToken token;
do
{
token = CTokenizer.GetToken(stream);
if (token.Kind == CppTokenKind.Eof)
{
break;
}
outTokens.Add(token);
} while (token.Kind != CppTokenKind.Eof);
}
}
break;
case TokenizerType.DoxygenCpp:
{
using (var lexer = new CppLexer(source, new TextPosition(0), source.Length))
{
var tokens = lexer.Tokenize();
foreach (var token in tokens)
{
if (token.Kind == CppTokenKind.Eof)
{
break;
}
var start = token.Position;
var end = new TextPosition(token.Position.Index + token.Length, token.Position.Line, token.Position.Column);
var value = source.Substring(start.Index, end.Index - start.Index);
outTokens.Add(new CToken(token.Kind, start, end, value));
}
}
}
break;
default:
throw new Exception($"Unsupported tokenizer type -> {tokenizerType}");
}
timer.Stop();
spans.Add(timer.Elapsed);
Console.WriteLine($"{tokenizerType} tokenizer[{iteration}/{numberOfIterationsPerType}] done, got {outTokens.Count()} tokens, took {timer.Elapsed.TotalMilliseconds} ms");
}
}
foreach (TokenizerType tokenizerType in tokenizerTypes)
{
List <TimeSpan> timeSpans = durations[tokenizerType];
TimeSpan minTime = GetMinTime(timeSpans);
TimeSpan maxTime = GetMaxTime(timeSpans);
TimeSpan avgTime = GetAvgTime(timeSpans);
Console.WriteLine($"{tokenizerType} tokenizer, min: {minTime}, max: {maxTime}, avg: {avgTime}, iterations: {numberOfIterationsPerType}");
}
#if false
// Compare tokens against each other
foreach (TokenizerType tokenizerTypeA in tokenizerTypes)
{
List <CToken> tokensA = tokenMap[tokenizerTypeA];
foreach (TokenizerType tokenizerTypeB in tokenizerTypes)
{
List <CToken> tokensB = tokenMap[tokenizerTypeB];
if (tokenizerTypeA != tokenizerTypeB)
{
CompareTokens(tokenizerTypeA, tokensA, tokenizerTypeB, tokensB);
}
}
}
#endif
string desktopPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
foreach (TokenizerType tokenizerType in tokenizerTypes)
{
string filename = $"tokenizer_{tokenizerType}.txt";
string singleFilePath = Path.Combine(desktopPath, filename);
List <CToken> tokens = tokenMap[tokenizerType];
using (StreamWriter writer = new StreamWriter(singleFilePath, false, Encoding.ASCII))
{
foreach (var token in tokens)
{
writer.Write(token);
writer.Write("\n");
}
}
}
Console.WriteLine("Press any key to exit");
Console.ReadKey();
return(0);
}
protected override bool LexNext(State hiddenState)
{
DoxygenState state = (DoxygenState)hiddenState;
state.Flags = StateFlags.None;
state.CurrentLineStartIndex = Buffer.StreamPosition;
do
{
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
switch (first)
{
case ' ':
case '\v':
case '\f':
case '\t':
Buffer.SkipSpacings(TextStream.SkipType.All);
break;
case '\r':
case '\n':
{
if (state.Flags.HasFlag(StateFlags.InsideBlock) && state.Flags.HasFlag(StateFlags.SingleLine))
{
Done(state);
return(true);
}
// @NOTE(final): Detect if our line content until the line break was empty
Debug.Assert(Buffer.StreamPosition >= state.CurrentLineStartIndex);
int len = Buffer.StreamPosition - state.CurrentLineStartIndex;
bool wasEmptyLine = Buffer.MatchCharacters(state.CurrentLineStartIndex, len, char.IsWhiteSpace) || (len == 0);
Buffer.StartLexeme();
Buffer.SkipLineBreaks(TextStream.SkipType.Single);
state.CurrentLineStartIndex = Buffer.StreamPosition;
PushToken(DoxygenTokenPool.Make(wasEmptyLine ? DoxygenTokenKind.EmptyLine : DoxygenTokenKind.EndOfLine, Buffer.LexemeRange, true));
}
break;
case '/':
{
if (second == '*')
{
// Multi line
if (DoxygenSyntax.MultiLineDocChars.Contains(third))
{
Debug.Assert(!state.Flags.HasFlag(StateFlags.InsideBlock));
Buffer.StartLexeme();
Buffer.AdvanceColumns(3);
state.Flags = StateFlags.InsideBlock;
if (third == '*')
{
char n3 = Buffer.Peek();
if (n3 == '/')
{
Buffer.AdvanceColumn();
return(PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartMulti, Buffer.LexemeRange, true)));
}
state.Flags |= StateFlags.JavaDoc;
}
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartMulti, Buffer.LexemeRange, true));
StartText(state);
continue;
}
else
{
// Just skip until normal multi-line comment ends
var r = CppLexer.LexMultiLineComment(Buffer, true);
if (!r.IsComplete)
{
AddError(Buffer.TextPosition, $"Unterminated multi-line comment, expect '*/' but got EOF", r.Kind.ToString());
return(false);
}
continue;
}
}
else if (second == '/')
{
// Single line
char n2 = Buffer.Peek(2);
if (DoxygenSyntax.SingleLineDocChars.Contains(n2))
{
Debug.Assert(!state.Flags.HasFlag(StateFlags.InsideBlock));
Buffer.StartLexeme();
Buffer.AdvanceColumns(3);
state.Flags = StateFlags.InsideBlock | StateFlags.SingleLine;
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartSingle, Buffer.LexemeRange, true));
StartText(state);
continue;
}
else
{
// Just skip until normal single-line comment ends
var r = CppLexer.LexSingleLineComment(Buffer, true);
if (!r.IsComplete)
{
AddError(Buffer.TextPosition, $"Unterminated single-line comment, expect linebreak but got EOF", r.Kind.ToString());
return(false);
}
continue;
}
}
else
{
Buffer.AdvanceColumn();
}
}
break;
case '*':
{
if (state.Flags.HasFlag(StateFlags.InsideBlock))
{
if (second == '/')
{
EndText(state);
Buffer.StartLexeme();
Buffer.AdvanceColumns(2);
state.Flags = StateFlags.None;
return(PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockEnd, Buffer.LexemeRange, true)));
}
else if (state.Flags.HasFlag(StateFlags.JavaDoc))
{
// Push single star token (java doc style)
EndText(state);
Buffer.StartLexeme();
Buffer.AdvanceColumn();
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockChars, Buffer.LexemeRange, true));
StartText(state);
state.CurrentLineStartIndex = Buffer.StreamPosition;
continue;
}
}
Buffer.AdvanceColumn();
}
break;
case '@':
case '\\':
{
if (state.Flags.HasFlag(StateFlags.InsideBlock))
{
EndText(state);
var commandResult = LexCommandTokens();
if (commandResult.IsValid)
{
if ("code".Equals(commandResult.CommandName))
{
if (!LexUntilCodeEnd(commandResult))
{
return(false);
}
}
}
StartText(state);
}
else
{
Buffer.AdvanceColumn();
}
}
break;
case TextStream.InvalidCharacter:
{
if (Buffer.IsEOF)
{
Done(state);
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.EOF, new TextRange(Buffer.TextPosition, 0), false));
return(false);
}
else
{
Buffer.AdvanceColumn();
}
}
break;
default:
{
Buffer.AdvanceColumn();
break;
}
}
} while (!Buffer.IsEOF);
Done(state);
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.EOF, new TextRange(Buffer.TextPosition, 0), false));
return(false);
}
private Dictionary <string, string> GetDefinedMacroses(IBuffer buffer, out bool isSurface)
{
isSurface = false;
var lexer = CppLexer.Create(buffer);
lexer.Start();
var definedMacroses = new Dictionary <string, string>();
var shaderTarget = HlslConstants.SHADER_TARGET_25;
while (lexer.TokenType != null)
{
var tokenType = lexer.TokenType;
if (tokenType is CppDirectiveTokenNodeType)
{
lexer.Advance();
var context = lexer.GetTokenText().TrimStart();
var(pragmaType, firstValue) = GetPragmaAndValue(context);
if (pragmaType.Equals("surface"))
{
isSurface = true;
}
// based on https://docs.unity3d.com/Manual/SL-ShaderPrograms.html
// We do not have any solution how we could handle multi_compile direcitves. It is complex task because
// a lot of different files could be produces from multi_compile combination
// Right now, we will consider first combination.
if (!firstValue.IsEmpty() && (pragmaType.Equals("multi_compile") || pragmaType.Equals("multi_compile_local") ||
pragmaType.Equals("shader_feature_local") || pragmaType.Equals("shader_feature")))
{
definedMacroses[firstValue] = "1";
}
if (pragmaType.Equals("target"))
{
var versionFromTarget = int.TryParse(firstValue.Replace(".", ""), out var result) ? result : HlslConstants.SHADER_TARGET_35;
shaderTarget = Math.Max(shaderTarget, versionFromTarget);
}
if (pragmaType.Equals("geometry"))
{
shaderTarget = Math.Max(shaderTarget, HlslConstants.SHADER_TARGET_40);
}
if (pragmaType.Equals("hull") || pragmaType.Equals("domain"))
{
shaderTarget = Math.Max(shaderTarget, HlslConstants.SHADER_TARGET_46);
}
// https://en.wikibooks.org/wiki/GLSL_Programming/Unity/Cookies
if (pragmaType.Equals("multi_compile_lightpass"))
{
// multi_compile_lightpass == multi_compile DIRECTIONAL, DIRECTIONAL_COOKIE, POINT, POINT_NOATT, POINT_COOKIE, SPOT
definedMacroses["DIRECTIONAL"] = "1";
}
// TODO: handle built-in https://docs.unity3d.com/Manual/SL-MultipleProgramVariants.html
// multi_compile_fwdbase, multi_compile_fwdadd, multi_compile_fwdadd_fullshadows, multi_compile_fog
// could not find information about that directives
}
lexer.Advance();
}
definedMacroses["SHADER_TARGET"] = shaderTarget.ToString();
return(definedMacroses);
}
public IEnumerable <CppFileLocation> GetIncludes(CppFileLocation cppFileLocation)
{
// PSI is not commited here
// TODO: cpp global cache should calculate cache only when PSI for file with cpp injects is committed.
var sourceFile = cppFileLocation.GetRandomSourceFile(mySolution);
var range = cppFileLocation.RootRange;
Assertion.Assert(range.IsValid, "range.IsValid");
var buffer = sourceFile.Document.Buffer;
var type = GetShaderProgramType(buffer, range.StartOffset);
var includeType = GetIncludeProgramType(type);
yield return(new CppFileLocation(myCppExternalModule, mySolution.SolutionDirectory.Combine(Utils.ShaderConfigFile)));
if (includeType != InjectedHlslProgramType.Uknown)
{
var includes = GetIncludesLocation(sourceFile, includeType);
foreach (var include in includes)
{
yield return(include);
}
}
var cgIncludeFolder = CgIncludeDirectoryTracker.GetCgIncludeFolderPath(myUnityVersion);
if (!cgIncludeFolder.ExistsDirectory)
{
yield break;
}
if (type == InjectedHlslProgramType.CGProgram)
{
var hlslSupport = cgIncludeFolder.Combine("HLSLSupport.cginc");
if (hlslSupport.ExistsFile)
{
yield return(new CppFileLocation(myCppExternalModule, hlslSupport));
}
var variables = cgIncludeFolder.Combine("UnityShaderVariables.cginc");
if (variables.ExistsFile)
{
yield return(new CppFileLocation(myCppExternalModule, variables));
}
}
var lexer = CppLexer.Create(ProjectedBuffer.Create(buffer, range));
lexer.Start();
while (lexer.TokenType != null)
{
var tokenType = lexer.TokenType;
if (tokenType is CppDirectiveTokenNodeType)
{
lexer.Advance();
var context = lexer.GetTokenText().TrimStart();
if (context.StartsWith("surface"))
{
var unityCG = cgIncludeFolder.Combine("UnityCG.cginc");
if (unityCG.ExistsFile)
{
yield return(new CppFileLocation(myCppExternalModule, unityCG));
}
var lighting = cgIncludeFolder.Combine("Lighting.cginc");
if (lighting.ExistsFile)
{
yield return(new CppFileLocation(myCppExternalModule, lighting));
}
var unityPbsLighting = cgIncludeFolder.Combine("UnityPBSLighting.cginc");
if (unityPbsLighting.ExistsFile)
{
yield return(new CppFileLocation(myCppExternalModule, unityPbsLighting));
}
var autoLight = cgIncludeFolder.Combine("AutoLight.cginc");
if (autoLight.ExistsFile)
{
yield return(new CppFileLocation(myCppExternalModule, autoLight));
}
break;
}
// TODO: optimization if we found vertex/fragment/geometry/hull/domain?
}
lexer.Advance();
}
}
