private TokenizeResult TokenizeCpp(string text, TextPosition pos, int length, bool allowDoxy) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = new Stopwatch(); timer.Restart(); List <CppToken> cppTokens = new List <CppToken>(); using (CppLexer cppLexer = new CppLexer(text, pos, length)) { cppTokens.AddRange(cppLexer.Tokenize()); result.AddErrors(cppLexer.LexErrors); } timer.Stop(); result.Stats.CppDuration += timer.Elapsed; foreach (CppToken token in cppTokens) { if (allowDoxy && (token.Kind == CppTokenKind.MultiLineCommentDoc || token.Kind == CppTokenKind.SingleLineCommentDoc)) { result.AddToken(token); using (TokenizeResult doxyRes = TokenizeDoxy(text, token.Position, token.Length)) { result.Stats.DoxyDuration += doxyRes.Stats.DoxyDuration; result.Stats.HtmlDuration += doxyRes.Stats.HtmlDuration; result.AddTokens(doxyRes.Tokens); result.AddErrors(doxyRes.Errors); } } else { result.AddToken(token); } } return(result); }
const string ErrMsgFormat = "-- line {0} col {1}: {2}"; // 0=line, 1=column, 2=text public CppParser(CppLexer lexer) { this.errors = lexer.Errors; errors.SynErr = new ErrorCodeProc(SynErr); this.lexer = (CppLexer)lexer; this.stack = new Stack <AstNode>(); }
public void Dispose() { errors = null; if (lexer != null) { lexer.Dispose(); } lexer = null; }
private ColourisedQuickInfoContent CreateColourisedContent(string content) { // TODO Investigate using C++'s colouriser instead of tokenising the content // ourselves. We might then be able to colour types and identifiers. var lexer = new CppLexer(); CppToken[] tokens = lexer.GetTokens(content); return(new ColourisedQuickInfoContent(tokens)); }
private CppParser BuildParser(string sourceCode) { // Parse the file var inputStream = new AntlrInputStream(sourceCode); var lexer = new CppLexer(inputStream); var tokenStream = new CommonTokenStream(lexer); var parser = new CppParser(tokenStream); // Setup error handling lexer.RemoveErrorListeners(); lexer.AddErrorListener(new LexerExceptionErrorListener()); parser.RemoveErrorListeners(); parser.AddErrorListener(new ParserExceptionErrorListener()); return(parser); }
private void RunTest(string sourceCode, int expectedToken) { // Parse the file var inputStream = new AntlrInputStream(sourceCode); var lexer = new CppLexer(inputStream); // Setup error handling lexer.RemoveErrorListeners(); lexer.AddErrorListener(new LexerExceptionErrorListener()); // Match all tokens var tokens = lexer.GetAllTokens(); // Verify we got the single expected token Assert.Equal(1, tokens.Count); Assert.Equal(expectedToken, tokens[0].Type); }
public static TranslationUnit GenerateAST(AntlrInputStream inputStream) { // Parse the file var lexer = new CppLexer(inputStream); var tokenStream = new CommonTokenStream(lexer); var parser = new CppParser(tokenStream); // var tokens = lexer.GetAllTokens().Select(token => lexer.Vocabulary.GetSymbolicName(token.Type)).ToList(); // Setup error handling lexer.RemoveErrorListeners(); lexer.AddErrorListener(new LexerExceptionErrorListener()); parser.RemoveErrorListeners(); parser.AddErrorListener(new ParserExceptionErrorListener()); // Parse the concrete syntax tree var translationUnit = parser.translationUnit(); // Convert the the abstract syntax tree var visitor = new ASTVisitor(); var ast = (TranslationUnit)visitor.Visit(translationUnit); return(ast); }
static int Main(string[] args) { if (args.Length < 1) { Console.Error.WriteLine("Missing filepath argument!"); return(-1); } string filePath = args[0]; string source = File.ReadAllText(filePath); TokenizerType[] tokenizerTypes = new[] { TokenizerType.Superpower, TokenizerType.DoxygenCpp, TokenizerType.CustomCpp }; const int numberOfIterationsPerType = 1; Dictionary <TokenizerType, List <TimeSpan> > durations = new Dictionary <TokenizerType, List <TimeSpan> >(); Dictionary <TokenizerType, List <CToken> > tokenMap = new Dictionary <TokenizerType, List <CToken> >(); foreach (TokenizerType tokenizerType in tokenizerTypes) { List <TimeSpan> spans = new List <TimeSpan>(); durations.Add(tokenizerType, spans); List <CToken> outTokens = new List <CToken>(); tokenMap.Add(tokenizerType, outTokens); for (int iteration = 1; iteration <= numberOfIterationsPerType; ++iteration) { outTokens.Clear(); Console.WriteLine($"{tokenizerType} tokenizer[{iteration}/{numberOfIterationsPerType}] start..."); Stopwatch timer = Stopwatch.StartNew(); switch (tokenizerType) { case TokenizerType.Superpower: { var tokenizer = new CSuperPowerTokenizer(); var tokens = tokenizer.Tokenize(source); foreach (var token in tokens) { if (token.Kind == CppTokenKind.Eof) { break; } var start = new TextPosition(token.Position.Absolute, token.Position.Line - 1, token.Position.Column - 1); var end = new TextPosition(token.Position.Absolute + token.Span.Length, token.Span.Position.Line - 1, token.Span.Position.Column - 1); var value = source.Substring(start.Index, end.Index - start.Index); outTokens.Add(new CToken(token.Kind, start, end, value)); } } break; case TokenizerType.CustomCpp: { using (var stream = new BasicTextStream(source, new TextPosition(0), source.Length)) { CToken token; do { token = CTokenizer.GetToken(stream); if (token.Kind == CppTokenKind.Eof) { break; } outTokens.Add(token); } while (token.Kind != CppTokenKind.Eof); } } break; case TokenizerType.DoxygenCpp: { using (var lexer = new CppLexer(source, new TextPosition(0), source.Length)) { var tokens = lexer.Tokenize(); foreach (var token in tokens) { if (token.Kind == CppTokenKind.Eof) { break; } var start = token.Position; var end = new TextPosition(token.Position.Index + token.Length, token.Position.Line, token.Position.Column); var value = source.Substring(start.Index, end.Index - start.Index); outTokens.Add(new CToken(token.Kind, start, end, value)); } } } break; default: throw new Exception($"Unsupported tokenizer type -> {tokenizerType}"); } timer.Stop(); spans.Add(timer.Elapsed); Console.WriteLine($"{tokenizerType} tokenizer[{iteration}/{numberOfIterationsPerType}] done, got {outTokens.Count()} tokens, took {timer.Elapsed.TotalMilliseconds} ms"); } } foreach (TokenizerType tokenizerType in tokenizerTypes) { List <TimeSpan> timeSpans = durations[tokenizerType]; TimeSpan minTime = GetMinTime(timeSpans); TimeSpan maxTime = GetMaxTime(timeSpans); TimeSpan avgTime = GetAvgTime(timeSpans); Console.WriteLine($"{tokenizerType} tokenizer, min: {minTime}, max: {maxTime}, avg: {avgTime}, iterations: {numberOfIterationsPerType}"); } #if false // Compare tokens against each other foreach (TokenizerType tokenizerTypeA in tokenizerTypes) { List <CToken> tokensA = tokenMap[tokenizerTypeA]; foreach (TokenizerType tokenizerTypeB in tokenizerTypes) { List <CToken> tokensB = tokenMap[tokenizerTypeB]; if (tokenizerTypeA != tokenizerTypeB) { CompareTokens(tokenizerTypeA, tokensA, tokenizerTypeB, tokensB); } } } #endif string desktopPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop); foreach (TokenizerType tokenizerType in tokenizerTypes) { string filename = $"tokenizer_{tokenizerType}.txt"; string singleFilePath = Path.Combine(desktopPath, filename); List <CToken> tokens = tokenMap[tokenizerType]; using (StreamWriter writer = new StreamWriter(singleFilePath, false, Encoding.ASCII)) { foreach (var token in tokens) { writer.Write(token); writer.Write("\n"); } } } Console.WriteLine("Press any key to exit"); Console.ReadKey(); return(0); }
protected override bool LexNext(State hiddenState) { DoxygenState state = (DoxygenState)hiddenState; state.Flags = StateFlags.None; state.CurrentLineStartIndex = Buffer.StreamPosition; do { char first = Buffer.Peek(); char second = Buffer.Peek(1); char third = Buffer.Peek(2); switch (first) { case ' ': case '\v': case '\f': case '\t': Buffer.SkipSpacings(TextStream.SkipType.All); break; case '\r': case '\n': { if (state.Flags.HasFlag(StateFlags.InsideBlock) && state.Flags.HasFlag(StateFlags.SingleLine)) { Done(state); return(true); } // @NOTE(final): Detect if our line content until the line break was empty Debug.Assert(Buffer.StreamPosition >= state.CurrentLineStartIndex); int len = Buffer.StreamPosition - state.CurrentLineStartIndex; bool wasEmptyLine = Buffer.MatchCharacters(state.CurrentLineStartIndex, len, char.IsWhiteSpace) || (len == 0); Buffer.StartLexeme(); Buffer.SkipLineBreaks(TextStream.SkipType.Single); state.CurrentLineStartIndex = Buffer.StreamPosition; PushToken(DoxygenTokenPool.Make(wasEmptyLine ? DoxygenTokenKind.EmptyLine : DoxygenTokenKind.EndOfLine, Buffer.LexemeRange, true)); } break; case '/': { if (second == '*') { // Multi line if (DoxygenSyntax.MultiLineDocChars.Contains(third)) { Debug.Assert(!state.Flags.HasFlag(StateFlags.InsideBlock)); Buffer.StartLexeme(); Buffer.AdvanceColumns(3); state.Flags = StateFlags.InsideBlock; if (third == '*') { char n3 = Buffer.Peek(); if (n3 == '/') { Buffer.AdvanceColumn(); return(PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartMulti, Buffer.LexemeRange, true))); } state.Flags |= StateFlags.JavaDoc; } PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartMulti, Buffer.LexemeRange, true)); StartText(state); continue; } else { // Just skip until normal multi-line comment ends var r = CppLexer.LexMultiLineComment(Buffer, true); if (!r.IsComplete) { AddError(Buffer.TextPosition, $"Unterminated multi-line comment, expect '*/' but got EOF", r.Kind.ToString()); return(false); } continue; } } else if (second == '/') { // Single line char n2 = Buffer.Peek(2); if (DoxygenSyntax.SingleLineDocChars.Contains(n2)) { Debug.Assert(!state.Flags.HasFlag(StateFlags.InsideBlock)); Buffer.StartLexeme(); Buffer.AdvanceColumns(3); state.Flags = StateFlags.InsideBlock | StateFlags.SingleLine; PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartSingle, Buffer.LexemeRange, true)); StartText(state); continue; } else { // Just skip until normal single-line comment ends var r = CppLexer.LexSingleLineComment(Buffer, true); if (!r.IsComplete) { AddError(Buffer.TextPosition, $"Unterminated single-line comment, expect linebreak but got EOF", r.Kind.ToString()); return(false); } continue; } } else { Buffer.AdvanceColumn(); } } break; case '*': { if (state.Flags.HasFlag(StateFlags.InsideBlock)) { if (second == '/') { EndText(state); Buffer.StartLexeme(); Buffer.AdvanceColumns(2); state.Flags = StateFlags.None; return(PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockEnd, Buffer.LexemeRange, true))); } else if (state.Flags.HasFlag(StateFlags.JavaDoc)) { // Push single star token (java doc style) EndText(state); Buffer.StartLexeme(); Buffer.AdvanceColumn(); PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockChars, Buffer.LexemeRange, true)); StartText(state); state.CurrentLineStartIndex = Buffer.StreamPosition; continue; } } Buffer.AdvanceColumn(); } break; case '@': case '\\': { if (state.Flags.HasFlag(StateFlags.InsideBlock)) { EndText(state); var commandResult = LexCommandTokens(); if (commandResult.IsValid) { if ("code".Equals(commandResult.CommandName)) { if (!LexUntilCodeEnd(commandResult)) { return(false); } } } StartText(state); } else { Buffer.AdvanceColumn(); } } break; case TextStream.InvalidCharacter: { if (Buffer.IsEOF) { Done(state); PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.EOF, new TextRange(Buffer.TextPosition, 0), false)); return(false); } else { Buffer.AdvanceColumn(); } } break; default: { Buffer.AdvanceColumn(); break; } } } while (!Buffer.IsEOF); Done(state); PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.EOF, new TextRange(Buffer.TextPosition, 0), false)); return(false); }
private Dictionary <string, string> GetDefinedMacroses(IBuffer buffer, out bool isSurface) { isSurface = false; var lexer = CppLexer.Create(buffer); lexer.Start(); var definedMacroses = new Dictionary <string, string>(); var shaderTarget = HlslConstants.SHADER_TARGET_25; while (lexer.TokenType != null) { var tokenType = lexer.TokenType; if (tokenType is CppDirectiveTokenNodeType) { lexer.Advance(); var context = lexer.GetTokenText().TrimStart(); var(pragmaType, firstValue) = GetPragmaAndValue(context); if (pragmaType.Equals("surface")) { isSurface = true; } // based on https://docs.unity3d.com/Manual/SL-ShaderPrograms.html // We do not have any solution how we could handle multi_compile direcitves. It is complex task because // a lot of different files could be produces from multi_compile combination // Right now, we will consider first combination. if (!firstValue.IsEmpty() && (pragmaType.Equals("multi_compile") || pragmaType.Equals("multi_compile_local") || pragmaType.Equals("shader_feature_local") || pragmaType.Equals("shader_feature"))) { definedMacroses[firstValue] = "1"; } if (pragmaType.Equals("target")) { var versionFromTarget = int.TryParse(firstValue.Replace(".", ""), out var result) ? result : HlslConstants.SHADER_TARGET_35; shaderTarget = Math.Max(shaderTarget, versionFromTarget); } if (pragmaType.Equals("geometry")) { shaderTarget = Math.Max(shaderTarget, HlslConstants.SHADER_TARGET_40); } if (pragmaType.Equals("hull") || pragmaType.Equals("domain")) { shaderTarget = Math.Max(shaderTarget, HlslConstants.SHADER_TARGET_46); } // https://en.wikibooks.org/wiki/GLSL_Programming/Unity/Cookies if (pragmaType.Equals("multi_compile_lightpass")) { // multi_compile_lightpass == multi_compile DIRECTIONAL, DIRECTIONAL_COOKIE, POINT, POINT_NOATT, POINT_COOKIE, SPOT definedMacroses["DIRECTIONAL"] = "1"; } // TODO: handle built-in https://docs.unity3d.com/Manual/SL-MultipleProgramVariants.html // multi_compile_fwdbase, multi_compile_fwdadd, multi_compile_fwdadd_fullshadows, multi_compile_fog // could not find information about that directives } lexer.Advance(); } definedMacroses["SHADER_TARGET"] = shaderTarget.ToString(); return(definedMacroses); }
public IEnumerable <CppFileLocation> GetIncludes(CppFileLocation cppFileLocation) { // PSI is not commited here // TODO: cpp global cache should calculate cache only when PSI for file with cpp injects is committed. var sourceFile = cppFileLocation.GetRandomSourceFile(mySolution); var range = cppFileLocation.RootRange; Assertion.Assert(range.IsValid, "range.IsValid"); var buffer = sourceFile.Document.Buffer; var type = GetShaderProgramType(buffer, range.StartOffset); var includeType = GetIncludeProgramType(type); yield return(new CppFileLocation(myCppExternalModule, mySolution.SolutionDirectory.Combine(Utils.ShaderConfigFile))); if (includeType != InjectedHlslProgramType.Uknown) { var includes = GetIncludesLocation(sourceFile, includeType); foreach (var include in includes) { yield return(include); } } var cgIncludeFolder = CgIncludeDirectoryTracker.GetCgIncludeFolderPath(myUnityVersion); if (!cgIncludeFolder.ExistsDirectory) { yield break; } if (type == InjectedHlslProgramType.CGProgram) { var hlslSupport = cgIncludeFolder.Combine("HLSLSupport.cginc"); if (hlslSupport.ExistsFile) { yield return(new CppFileLocation(myCppExternalModule, hlslSupport)); } var variables = cgIncludeFolder.Combine("UnityShaderVariables.cginc"); if (variables.ExistsFile) { yield return(new CppFileLocation(myCppExternalModule, variables)); } } var lexer = CppLexer.Create(ProjectedBuffer.Create(buffer, range)); lexer.Start(); while (lexer.TokenType != null) { var tokenType = lexer.TokenType; if (tokenType is CppDirectiveTokenNodeType) { lexer.Advance(); var context = lexer.GetTokenText().TrimStart(); if (context.StartsWith("surface")) { var unityCG = cgIncludeFolder.Combine("UnityCG.cginc"); if (unityCG.ExistsFile) { yield return(new CppFileLocation(myCppExternalModule, unityCG)); } var lighting = cgIncludeFolder.Combine("Lighting.cginc"); if (lighting.ExistsFile) { yield return(new CppFileLocation(myCppExternalModule, lighting)); } var unityPbsLighting = cgIncludeFolder.Combine("UnityPBSLighting.cginc"); if (unityPbsLighting.ExistsFile) { yield return(new CppFileLocation(myCppExternalModule, unityPbsLighting)); } var autoLight = cgIncludeFolder.Combine("AutoLight.cginc"); if (autoLight.ExistsFile) { yield return(new CppFileLocation(myCppExternalModule, autoLight)); } break; } // TODO: optimization if we found vertex/fragment/geometry/hull/domain? } lexer.Advance(); } }