protected bool ParsePPSymbol(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt) { var word = FGParser.ScanIdentifierOrKeyword(line, ref startAt); if (word == null) { return(true); } word.tokenKind = SyntaxToken.Kind.PreprocessorSymbol; formatedLine.tokens.Add(word); word.formatedLine = formatedLine; if (word.text == "true") { return(true); } if (word.text == "false") { return(false); } if (scriptDefines == null) { scriptDefines = new HashSet <string>(UnityEditor.EditorUserBuildSettings.activeScriptCompilationDefines); } var isDefined = scriptDefines.Contains(word.text); return(isDefined); }
private new SyntaxToken ScanIdentifierOrKeyword(string line, ref int startAt) { var token = FGParser.ScanIdentifierOrKeyword(line, ref startAt); if (token != null && token.tokenKind == SyntaxToken.Kind.Keyword && !IsKeywordOrBuiltInType(token.text)) { token.tokenKind = SyntaxToken.Kind.Identifier; } return(token); }
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine) { var tokens = formatedLine.tokens ?? new List <SyntaxToken>(); formatedLine.tokens = tokens; tokens.Clear(); int startAt = 0; int length = line.Length; SyntaxToken token; SyntaxToken ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } if (formatedLine.blockState == FGTextBuffer.BlockState.None && startAt < length && line[startAt] == '#') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Preprocessor, "#") { formatedLine = formatedLine }); ++startAt; ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } var error = false; var commentsOnly = false; var preprocessorCommentsAllowed = true; token = ScanWord(line, ref startAt); if (Array.BinarySearch(preprocessorKeywords, token.text) < 0) { token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected; tokens.Add(token); token.formatedLine = formatedLine; error = true; } else { token.tokenKind = SyntaxToken.Kind.Preprocessor; tokens.Add(token); token.formatedLine = formatedLine; ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } if (token.text == "if") { bool active = ParsePPOrExpression(line, formatedLine, ref startAt); bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (active && !inactiveParent) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.If); commentsOnly = true; } else { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveIf); commentsOnly = true; } } else if (token.text == "elif") { bool active = ParsePPOrExpression(line, formatedLine, ref startAt); bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif); } else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf) { inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (active && !inactiveParent) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Elif); } else { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif); } } else { token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; } } else if (token.text == "else") { if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse); } else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif) { bool inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (inactiveParent) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse); } else { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Else); } } else { token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; } } else if (token.text == "endif") { if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Else || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElse) { CloseRegion(formatedLine); } else { token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; } } else if (token.text == "region") { var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (inactive) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveRegion); } else { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Region); } preprocessorCommentsAllowed = false; } else if (token.text == "endregion") { if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Region || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveRegion) { CloseRegion(formatedLine); } else { token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; } preprocessorCommentsAllowed = false; } else if (token.text == "define" || token.text == "undef") { var symbol = FGParser.ScanIdentifierOrKeyword(line, ref startAt); if (symbol != null && symbol.text != "true" && symbol.text != "false") { symbol.tokenKind = SyntaxToken.Kind.PreprocessorSymbol; formatedLine.tokens.Add(symbol); symbol.formatedLine = formatedLine; scriptDefinesChanged = true; var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (!inactive) { if (token.text == "define") { if (!scriptDefines.Contains(symbol.text)) { scriptDefines.Add(symbol.text); //scriptDefinesChanged = true; } } else { if (scriptDefines.Contains(symbol.text)) { scriptDefines.Remove(symbol.text); //scriptDefinesChanged = true; } } } } } else if (token.text == "error" || token.text == "warning") { preprocessorCommentsAllowed = false; } } if (!preprocessorCommentsAllowed) { ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } if (startAt < length) { var textArgument = line.Substring(startAt); textArgument.TrimEnd(new [] { ' ', '\t' }); tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, textArgument) { formatedLine = formatedLine }); startAt = length - textArgument.Length; if (startAt < length) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Whitespace, line.Substring(startAt)) { formatedLine = formatedLine }); } } return; } while (startAt < length) { ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; continue; } var firstChar = line[startAt]; if (startAt < length - 1 && firstChar == '/' && line[startAt + 1] == '/') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); break; } else if (commentsOnly) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorCommentExpected, line.Substring(startAt)) { formatedLine = formatedLine }); break; } if (char.IsLetterOrDigit(firstChar) || firstChar == '_') { token = ScanWord(line, ref startAt); token.tokenKind = SyntaxToken.Kind.PreprocessorArguments; tokens.Add(token); token.formatedLine = formatedLine; } else if (firstChar == '"') { token = ScanStringLiteral(line, ref startAt); token.tokenKind = SyntaxToken.Kind.PreprocessorArguments; tokens.Add(token); token.formatedLine = formatedLine; } else if (firstChar == '\'') { token = ScanCharLiteral(line, ref startAt); token.tokenKind = SyntaxToken.Kind.PreprocessorArguments; tokens.Add(token); token.formatedLine = formatedLine; } else { token = new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, firstChar.ToString()) { formatedLine = formatedLine }; tokens.Add(token); ++startAt; } if (error) { token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected; } } return; } var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; while (startAt < length) { switch (formatedLine.blockState) { case FGTextBuffer.BlockState.None: ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; continue; } if (inactiveLine) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } if (line[startAt] == '/' && startAt < length - 1) { if (line[startAt + 1] == '/') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "//") { formatedLine = formatedLine }); startAt += 2; tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } else if (line[startAt + 1] == '*') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "/*") { formatedLine = formatedLine }); startAt += 2; formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock; break; } } if (line[startAt] == '\'') { token = ScanCharLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } if (line[startAt] == '\"') { token = ScanStringLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } if (startAt < length - 1 && line[startAt] == '@' && line[startAt + 1] == '\"') { token = new SyntaxToken(SyntaxToken.Kind.VerbatimStringBegin, line.Substring(startAt, 2)) { formatedLine = formatedLine }; tokens.Add(token); startAt += 2; formatedLine.blockState = FGTextBuffer.BlockState.StringBlock; break; } if (line[startAt] >= '0' && line[startAt] <= '9' || startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9') { token = ScanNumericLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } token = ScanIdentifierOrKeyword(line, ref startAt); if (token != null) { tokens.Add(token); token.formatedLine = formatedLine; break; } // Multi-character operators / punctuators // "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "??", "+=", "-=", "*=", "/=", "%=", // "&=", "|=", "^=", "<<=", ">>=", "=>", "::" var punctuatorStart = startAt++; if (startAt < line.Length) { switch (line[punctuatorStart]) { case '?': if (line[startAt] == '?') { ++startAt; } break; case '+': if (line[startAt] == '+' || line[startAt] == '=') { ++startAt; } break; case '-': if (line[startAt] == '-' || line[startAt] == '=') { ++startAt; } break; case '<': if (line[startAt] == '=') { ++startAt; } else if (line[startAt] == '<') { ++startAt; if (startAt < line.Length && line[startAt] == '=') { ++startAt; } } break; case '>': if (line[startAt] == '=') { ++startAt; } //else if (startAt < line.Length && line[startAt] == '>') //{ // ++startAt; // if (line[startAt] == '=') // ++startAt; //} break; case '=': if (line[startAt] == '=' || line[startAt] == '>') { ++startAt; } break; case '&': if (line[startAt] == '=' || line[startAt] == '&') { ++startAt; } break; case '|': if (line[startAt] == '=' || line[startAt] == '|') { ++startAt; } break; case '*': case '/': case '%': case '^': case '!': if (line[startAt] == '=') { ++startAt; } break; case ':': if (line[startAt] == ':') { ++startAt; } break; } } tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart)) { formatedLine = formatedLine }); break; case FGTextBuffer.BlockState.CommentBlock: int commentBlockEnd = line.IndexOf("*/", startAt, StringComparison.Ordinal); if (commentBlockEnd == -1) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; } else { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt)) { formatedLine = formatedLine }); startAt = commentBlockEnd + 2; formatedLine.blockState = FGTextBuffer.BlockState.None; } break; case FGTextBuffer.BlockState.StringBlock: int i = startAt; int closingQuote = line.IndexOf('\"', startAt); while (closingQuote != -1 && closingQuote < length - 1 && line[closingQuote + 1] == '\"') { i = closingQuote + 2; closingQuote = line.IndexOf('\"', i); } if (closingQuote == -1) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; } else { tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt)) { formatedLine = formatedLine }); startAt = closingQuote; tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 1)) { formatedLine = formatedLine }); ++startAt; formatedLine.blockState = FGTextBuffer.BlockState.None; } break; } } }