public virtual void LexLine(int currentLine, FormatedLine formatedLine) { formatedLine.index = currentLine; if (parserThread != null) { parserThread.Join(); } parserThread = null; string textLine = textBuffer.lines[currentLine]; var lineTokens = formatedLine.tokens ?? new List <SyntaxToken>(); lineTokens.Clear(); formatedLine.tokens = lineTokens; if (!string.IsNullOrEmpty(textLine)) { //Tokenize(lineTokens, textLine, ref formatedLine.blockState); lineTokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, textLine) { style = textBuffer.styles.normalStyle, formatedLine = formatedLine }); var lineWidth = textBuffer.CharIndexToColumn(textLine.Length, currentLine); if (lineWidth > textBuffer.longestLine) { textBuffer.longestLine = lineWidth; } } }
protected bool ParsePPSymbol(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt) { var word = FGParser.ScanIdentifierOrKeyword(line, ref startAt); if (word == null) { return(true); } word.tokenKind = SyntaxToken.Kind.PreprocessorSymbol; formatedLine.tokens.Add(word); word.formatedLine = formatedLine; if (word.text == "true") { return(true); } if (word.text == "false") { return(false); } if (scriptDefines == null) { scriptDefines = new HashSet <string>(UnityEditor.EditorUserBuildSettings.activeScriptCompilationDefines); } var isDefined = scriptDefines.Contains(word.text); return(isDefined); }
protected bool ParsePPPrimaryExpression(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt) { if (line[startAt] == '(') { formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, "(") { formatedLine = formatedLine }); ++startAt; var ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } var result = ParsePPOrExpression(line, formatedLine, ref startAt); if (startAt >= line.Length) { //TODO: Insert missing token return(result); } if (line[startAt] == ')') { formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, ")") { formatedLine = formatedLine }); ++startAt; ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } return(result); } //TODO: Insert missing token return(result); } var symbolResult = ParsePPSymbol(line, formatedLine, ref startAt); var ws2 = ScanWhitespace(line, ref startAt); if (ws2 != null) { formatedLine.tokens.Add(ws2); ws2.formatedLine = formatedLine; } return(symbolResult); }
protected void OpenRegion(FGTextBuffer.FormatedLine formatedLine, FGTextBuffer.RegionTree.Kind regionKind) { var parentRegion = formatedLine.regionTree; FGTextBuffer.RegionTree reuseRegion = null; switch (regionKind) { case FGTextBuffer.RegionTree.Kind.Else: case FGTextBuffer.RegionTree.Kind.Elif: case FGTextBuffer.RegionTree.Kind.InactiveElse: case FGTextBuffer.RegionTree.Kind.InactiveElif: parentRegion = parentRegion.parent; break; } if (parentRegion.children != null) { for (var i = parentRegion.children.Count; i-- > 0;) { if (parentRegion.children[i].line == formatedLine) { reuseRegion = parentRegion.children[i]; break; } } } if (reuseRegion != null) { if (reuseRegion.kind == regionKind) { formatedLine.regionTree = reuseRegion; return; } reuseRegion.parent = null; parentRegion.children.Remove(reuseRegion); } formatedLine.regionTree = new FGTextBuffer.RegionTree { parent = parentRegion, kind = regionKind, line = formatedLine, }; if (parentRegion.children == null) { parentRegion.children = new List <FGTextBuffer.RegionTree>(); } parentRegion.children.Add(formatedLine.regionTree); }
protected bool ParsePPEqualityExpression(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt) { if (startAt >= line.Length) { //TODO: Insert missing token return(true); } var lhs = ParsePPUnaryExpression(line, formatedLine, ref startAt); var ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } if (startAt + 1 < line.Length && (line[startAt] == '=' || line[startAt + 1] == '!') && line[startAt + 1] == '=') { var equality = line[startAt] == '='; formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, equality ? "==" : "!=") { formatedLine = formatedLine }); startAt += 2; ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } var rhs = ParsePPEqualityExpression(line, formatedLine, ref startAt); ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } return(equality ? lhs == rhs : lhs != rhs); } return(lhs); }
protected bool ParsePPAndExpression(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt) { if (startAt >= line.Length) { //TODO: Insert missing token return(true); } var lhs = ParsePPEqualityExpression(line, formatedLine, ref startAt); var ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } if (startAt + 1 < line.Length && line[startAt] == '&' && line[startAt + 1] == '&') { formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, "&&") { formatedLine = formatedLine }); startAt += 2; ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } var rhs = ParsePPAndExpression(line, formatedLine, ref startAt); ws = ScanWhitespace(line, ref startAt); if (ws != null) { formatedLine.tokens.Add(ws); ws.formatedLine = formatedLine; } return(lhs && rhs); } return(lhs); }
protected virtual void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine) { }
protected void CloseRegion(FGTextBuffer.FormatedLine formatedLine) { formatedLine.regionTree = formatedLine.regionTree.parent; }
public override void LexLine(int currentLine, FGTextBuffer.FormatedLine formatedLine) { formatedLine.index = currentLine; if (parserThread != null) { parserThread.Join(); } parserThread = null; string textLine = textBuffer.lines[currentLine]; //Stopwatch sw1 = new Stopwatch(); //Stopwatch sw2 = new Stopwatch(); if (currentLine == 0) { var defaultScriptDefines = UnityEditor.EditorUserBuildSettings.activeScriptCompilationDefines; if (scriptDefines == null || !scriptDefines.SetEquals(defaultScriptDefines)) { if (scriptDefines == null) { scriptDefines = new HashSet <string>(defaultScriptDefines); } else { scriptDefines.Clear(); scriptDefines.UnionWith(defaultScriptDefines); } } } //sw2.Start(); Tokenize(textLine, formatedLine); // syntaxTree.SetLineTokens(currentLine, lineTokens); var lineTokens = formatedLine.tokens; if (textLine.Length == 0) { formatedLine.tokens.Clear(); } else if (textBuffer.styles != null) { var lineWidth = textBuffer.CharIndexToColumn(textLine.Length, currentLine); if (lineWidth > textBuffer.longestLine) { textBuffer.longestLine = lineWidth; } for (var i = 0; i < lineTokens.Count; ++i) { var token = lineTokens[i]; switch (token.tokenKind) { case SyntaxToken.Kind.Whitespace: case SyntaxToken.Kind.Missing: token.style = textBuffer.styles.normalStyle; break; case SyntaxToken.Kind.Punctuator: token.style = IsOperator(token.text) ? textBuffer.styles.operatorStyle : textBuffer.styles.punctuatorStyle; break; case SyntaxToken.Kind.Keyword: if (IsBuiltInType(token.text)) { if (token.text == "string" || token.text == "object") { token.style = textBuffer.styles.builtInRefTypeStyle; } else { token.style = textBuffer.styles.builtInValueTypeStyle; } } else { token.style = textBuffer.styles.keywordStyle; } break; case SyntaxToken.Kind.Identifier: if (IsBuiltInLiteral(token.text)) { token.style = textBuffer.styles.builtInLiteralsStyle; token.tokenKind = SyntaxToken.Kind.BuiltInLiteral; } else if (IsUnityType(token.text)) { token.style = textBuffer.styles.referenceTypeStyle; } else { token.style = textBuffer.styles.normalStyle; } break; case SyntaxToken.Kind.IntegerLiteral: case SyntaxToken.Kind.RealLiteral: token.style = textBuffer.styles.constantStyle; break; case SyntaxToken.Kind.Comment: var regionKind = formatedLine.regionTree.kind; var inactiveLine = regionKind > FGTextBuffer.RegionTree.Kind.LastActive; token.style = inactiveLine ? textBuffer.styles.inactiveCodeStyle : textBuffer.styles.commentStyle; break; case SyntaxToken.Kind.Preprocessor: token.style = textBuffer.styles.preprocessorStyle; break; case SyntaxToken.Kind.PreprocessorSymbol: token.style = textBuffer.styles.defineSymbols; break; case SyntaxToken.Kind.PreprocessorArguments: case SyntaxToken.Kind.PreprocessorCommentExpected: case SyntaxToken.Kind.PreprocessorDirectiveExpected: case SyntaxToken.Kind.PreprocessorUnexpectedDirective: token.style = textBuffer.styles.normalStyle; break; case SyntaxToken.Kind.CharLiteral: case SyntaxToken.Kind.StringLiteral: case SyntaxToken.Kind.VerbatimStringBegin: case SyntaxToken.Kind.VerbatimStringLiteral: token.style = textBuffer.styles.stringStyle; break; } lineTokens[i] = token; } } }
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine) { var tokens = new List <SyntaxToken>(); formatedLine.tokens = tokens; int startAt = 0; int length = line.Length; SyntaxToken token; SyntaxToken ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } if (formatedLine.blockState == FGTextBuffer.BlockState.None && startAt < length && line[startAt] == '#') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Preprocessor, "#") { formatedLine = formatedLine }); ++startAt; //ws = ScanWhitespace(line, ref startAt); //if (ws != null) //{ // tokens.Add(ws); // ws.formatedLine = formatedLine; //} var error = false; var commentsOnly = false; var preprocessorCommentsAllowed = true; token = ScanWord(line, ref startAt); if (Array.BinarySearch(preprocessorKeywords, token.text) < 0) { token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected; tokens.Add(token); token.formatedLine = formatedLine; error = true; } else { token.tokenKind = SyntaxToken.Kind.Preprocessor; tokens.Add(token); token.formatedLine = formatedLine; ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } if (token.text == "if") { bool active = ParsePPOrExpression(line, formatedLine, ref startAt); bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (active && !inactiveParent) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.If); commentsOnly = true; } else { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveIf); commentsOnly = true; } } else if (token.text == "elif") { bool active = ParsePPOrExpression(line, formatedLine, ref startAt); bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif); } else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf) { inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (active && !inactiveParent) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Elif); } else { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif); } } else { token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; } } else if (token.text == "else") { if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse); } else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif) { bool inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive; if (inactiveParent) { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse); } else { OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Else); } } else { token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; } } else if (token.text == "endif") { if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Else || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif || formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElse) { CloseRegion(formatedLine); } else { token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; } } //else if (token.text == "region") //{ // var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; // if (inactive) // { // OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveRegion); // } // else // { // OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Region); // } // preprocessorCommentsAllowed = false; //} //else if (token.text == "endregion") //{ // if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Region || // formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveRegion) // { // CloseRegion(formatedLine); // } // else // { // token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective; // } // preprocessorCommentsAllowed = false; //} //else if (token.text == "define" || token.text == "undef") //{ // var symbol = FGParser.ScanIdentifierOrKeyword(line, ref startAt); // if (symbol != null && symbol.text != "true" && symbol.text != "false") // { // symbol.tokenKind = SyntaxToken.Kind.PreprocessorSymbol; // formatedLine.tokens.Add(symbol); // symbol.formatedLine = formatedLine; // scriptDefinesChanged = true; // var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; // if (!inactive) // { // if (token.text == "define") // { // if (!scriptDefines.Contains(symbol.text)) // { // scriptDefines.Add(symbol.text); // //scriptDefinesChanged = true; // } // } // else // { // if (scriptDefines.Contains(symbol.text)) // { // scriptDefines.Remove(symbol.text); // //scriptDefinesChanged = true; // } // } // } // } //} //else if (token.text == "error" || token.text == "warning") //{ // preprocessorCommentsAllowed = false; //} } if (!preprocessorCommentsAllowed) { ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } if (startAt < length) { var textArgument = line.Substring(startAt); textArgument.TrimEnd(new [] { ' ', '\t' }); tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, textArgument) { formatedLine = formatedLine }); startAt = length - textArgument.Length; if (startAt < length) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Whitespace, line.Substring(startAt)) { formatedLine = formatedLine }); } } return; } while (startAt < length) { ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; continue; } var firstChar = line[startAt]; if (startAt < length - 1 && firstChar == '/' && line[startAt + 1] == '/') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); break; } else if (commentsOnly) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorCommentExpected, line.Substring(startAt)) { formatedLine = formatedLine }); break; } if (char.IsLetterOrDigit(firstChar) || firstChar == '_') { token = ScanWord(line, ref startAt); token.tokenKind = SyntaxToken.Kind.PreprocessorArguments; tokens.Add(token); token.formatedLine = formatedLine; } else if (firstChar == '"') { token = ScanStringLiteral(line, ref startAt); token.tokenKind = SyntaxToken.Kind.PreprocessorArguments; tokens.Add(token); token.formatedLine = formatedLine; } else if (firstChar == '\'') { token = ScanCharLiteral(line, ref startAt); token.tokenKind = SyntaxToken.Kind.PreprocessorArguments; tokens.Add(token); token.formatedLine = formatedLine; } else { token = new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, firstChar.ToString()) { formatedLine = formatedLine }; tokens.Add(token); ++startAt; } if (error) { token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected; } } return; } var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; while (startAt < length) { switch (formatedLine.blockState) { case FGTextBuffer.BlockState.None: ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; continue; } if (inactiveLine) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } if (line[startAt] == '/' && startAt < length - 1) { if (line[startAt + 1] == '/') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "//") { formatedLine = formatedLine }); startAt += 2; tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } else if (line[startAt + 1] == '*') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "/*") { formatedLine = formatedLine }); startAt += 2; formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock; break; } } if (line[startAt] == '\'') { token = ScanCharLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } if (line[startAt] == '\"') { token = ScanStringLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } //if (startAt < length - 1 && line[startAt] == '@' && line[startAt + 1] == '\"') //{ // token = new SyntaxToken(SyntaxToken.Kind.VerbatimStringBegin, line.Substring(startAt, 2)) { formatedLine = formatedLine }; // tokens.Add(token); // startAt += 2; // formatedLine.blockState = FGTextBuffer.BlockState.StringBlock; // break; //} if (line[startAt] >= '0' && line[startAt] <= '9' || startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9') { token = ScanNumericLiteral_JS(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } token = ScanIdentifierOrKeyword(line, ref startAt); if (token != null) { tokens.Add(token); token.formatedLine = formatedLine; break; } // Multi-character operators / punctuators // "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "??", "+=", "-=", "*=", "/=", "%=", // "&=", "|=", "^=", "<<=", ">>=", "=>", "::", "->", "^=" var punctuatorStart = startAt++; if (startAt < line.Length) { switch (line[punctuatorStart]) { case '?': if (line[startAt] == '?') { ++startAt; } break; case '+': if (line[startAt] == '+' || line[startAt] == '=' || line[startAt] == '>') { ++startAt; } break; case '-': if (line[startAt] == '-' || line[startAt] == '=') { ++startAt; } break; case '<': if (line[startAt] == '=') { ++startAt; } else if (line[startAt] == '<') { ++startAt; if (startAt < line.Length && line[startAt] == '=') { ++startAt; } } break; case '>': if (line[startAt] == '=') { ++startAt; } else if (startAt < line.Length && line[startAt] == '>') { ++startAt; if (line[startAt] == '=') { ++startAt; } } break; case '=': if (line[startAt] == '=' || line[startAt] == '>') { ++startAt; } break; case '&': if (line[startAt] == '=' || line[startAt] == '&') { ++startAt; } break; case '|': if (line[startAt] == '=' || line[startAt] == '|') { ++startAt; } break; case '*': case '/': case '%': case '^': case '!': if (line[startAt] == '=') { ++startAt; } break; case ':': if (line[startAt] == ':') { ++startAt; } break; } } tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart)) { formatedLine = formatedLine }); break; case FGTextBuffer.BlockState.CommentBlock: int commentBlockEnd = line.IndexOf("*/", startAt, StringComparison.Ordinal); if (commentBlockEnd == -1) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; } else { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt)) { formatedLine = formatedLine }); startAt = commentBlockEnd + 2; formatedLine.blockState = FGTextBuffer.BlockState.None; } break; //case FGTextBuffer.BlockState.StringBlock: // int i = startAt; // int closingQuote = line.IndexOf('\"', startAt); // while (closingQuote != -1 && closingQuote < length - 1 && line[closingQuote + 1] == '\"') // { // i = closingQuote + 2; // closingQuote = line.IndexOf('\"', i); // } // if (closingQuote == -1) // { // tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt)) { formatedLine = formatedLine }); // startAt = length; // } // else // { // tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt)) { formatedLine = formatedLine }); // startAt = closingQuote; // tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 1)) { formatedLine = formatedLine }); // ++startAt; // formatedLine.blockState = FGTextBuffer.BlockState.None; // } // break; } } }
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine) { var tokens = formatedLine.tokens ?? new List <SyntaxToken>(); formatedLine.tokens = tokens; tokens.Clear(); int startAt = 0; int length = line.Length; SyntaxToken token; SyntaxToken ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; while (startAt < length) { switch (formatedLine.blockState) { case FGTextBuffer.BlockState.None: ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; continue; } if (inactiveLine) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } //注释--、注释块开始--[[ if (line[startAt] == '-' && startAt < length - 1) { if (startAt + 3 <= length - 1 && line.Substring(startAt, 4) == "--[[") { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "--[[") { formatedLine = formatedLine }); startAt += 4; formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock; break; } else if (line[startAt + 1] == '-') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "--") { formatedLine = formatedLine }); startAt += 2; tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } } if (line[startAt] == '\'') { token = ScanCharLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } if (line[startAt] == '\"') { token = ScanStringLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } if (line[startAt] >= '0' && line[startAt] <= '9' || startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9') { token = ScanNumericLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } token = ScanIdentifierOrKeyword(line, ref startAt); if (token != null) { tokens.Add(token); token.formatedLine = formatedLine; break; } var punctuatorStart = startAt++; if (startAt < line.Length) { switch (line[punctuatorStart]) { case '?': if (line[startAt] == '?') { ++startAt; } break; case '+': if (line[startAt] == '+' || line[startAt] == '=') { ++startAt; } break; case '-': if (line[startAt] == '-' || line[startAt] == '=') { ++startAt; } break; case '<': if (line[startAt] == '=') { ++startAt; } else if (line[startAt] == '<') { ++startAt; if (startAt < line.Length && line[startAt] == '=') { ++startAt; } } break; case '>': if (line[startAt] == '=') { ++startAt; } //else if (startAt < line.Length && line[startAt] == '>') //{ // ++startAt; // if (line[startAt] == '=') // ++startAt; //} break; case '=': if (line[startAt] == '=' || line[startAt] == '>') { ++startAt; } break; case '&': if (line[startAt] == '=' || line[startAt] == '&') { ++startAt; } break; case '|': if (line[startAt] == '=' || line[startAt] == '|') { ++startAt; } break; case '*': case '/': case '%': case '^': case '!': if (line[startAt] == '=') { ++startAt; } break; case ':': if (line[startAt] == ':') { ++startAt; } break; } } tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart)) { formatedLine = formatedLine }); break; case FGTextBuffer.BlockState.CommentBlock: //注释 int commentBlockEnd = line.IndexOf("]]", startAt, StringComparison.Ordinal); if (commentBlockEnd == -1) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; } else { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt)) { formatedLine = formatedLine }); startAt = commentBlockEnd + 2; formatedLine.blockState = FGTextBuffer.BlockState.None; } break; case FGTextBuffer.BlockState.StringBlock: int i = startAt; int closingQuote = line.IndexOf('\"', startAt); while (closingQuote != -1 && closingQuote < length - 1 && line[closingQuote + 1] == '\"') { i = closingQuote + 2; closingQuote = line.IndexOf('\"', i); } if (closingQuote == -1) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; } else { tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt)) { formatedLine = formatedLine }); startAt = closingQuote; tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 1)) { formatedLine = formatedLine }); ++startAt; formatedLine.blockState = FGTextBuffer.BlockState.None; } break; } } }
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine) { var tokens = new List <SyntaxToken>(); formatedLine.tokens = tokens; int startAt = 0; int length = line.Length; SyntaxToken token; SyntaxToken ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; } var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive; while (startAt < length) { switch (formatedLine.blockState) { case FGTextBuffer.BlockState.None: ws = ScanWhitespace(line, ref startAt); if (ws != null) { tokens.Add(ws); ws.formatedLine = formatedLine; continue; } if (inactiveLine) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } if (line[startAt] == '#') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "#") { formatedLine = formatedLine }); ++startAt; tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } if (line[startAt] == '/' && startAt < length - 1) { if (line[startAt + 1] == '/') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "//") { formatedLine = formatedLine }); startAt += 2; tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; break; } else if (line[startAt + 1] == '*') { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "/*") { formatedLine = formatedLine }); startAt += 2; formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock; break; } } if (line[startAt] == '\'') { token = ScanCharLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } if (line[startAt] == '\"') { if (startAt < length - 2 && line[startAt + 1] == '\"' && line[startAt + 2] == '\"') { token = new SyntaxToken(SyntaxToken.Kind.VerbatimStringBegin, line.Substring(startAt, 3)) { formatedLine = formatedLine }; tokens.Add(token); startAt += 3; formatedLine.blockState = FGTextBuffer.BlockState.StringBlock; break; } token = ScanStringLiteral(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } if (line[startAt] >= '0' && line[startAt] <= '9' || startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9') { token = ScanNumericLiteral_JS(line, ref startAt); tokens.Add(token); token.formatedLine = formatedLine; break; } token = ScanIdentifierOrKeyword(line, ref startAt); if (token != null) { tokens.Add(token); token.formatedLine = formatedLine; break; } // Multi-character operators / punctuators // "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "??", "+=", "-=", "*=", "/=", "%=", // "&=", "|=", "^=", "<<=", ">>=", "=>", "::", "->", "^=" var punctuatorStart = startAt++; if (startAt < line.Length) { switch (line[punctuatorStart]) { case '?': if (line[startAt] == '?') { ++startAt; } break; case '+': if (line[startAt] == '+' || line[startAt] == '=' || line[startAt] == '>') { ++startAt; } break; case '-': if (line[startAt] == '-' || line[startAt] == '=') { ++startAt; } break; case '<': if (line[startAt] == '=') { ++startAt; } else if (line[startAt] == '<') { ++startAt; if (startAt < line.Length && line[startAt] == '=') { ++startAt; } } break; case '>': if (line[startAt] == '=') { ++startAt; } else if (startAt < line.Length && line[startAt] == '>') { ++startAt; if (line[startAt] == '=') { ++startAt; } } break; case '=': if (line[startAt] == '=' || line[startAt] == '>') { ++startAt; } break; case '&': if (line[startAt] == '=' || line[startAt] == '&') { ++startAt; } break; case '|': if (line[startAt] == '=' || line[startAt] == '|') { ++startAt; } break; case '*': case '/': case '%': case '^': case '!': if (line[startAt] == '=') { ++startAt; } break; case ':': if (line[startAt] == ':') { ++startAt; } break; } } tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart)) { formatedLine = formatedLine }); break; case FGTextBuffer.BlockState.CommentBlock: int commentBlockEnd = line.IndexOf("*/", startAt, StringComparison.Ordinal); if (commentBlockEnd == -1) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; } else { tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt)) { formatedLine = formatedLine }); startAt = commentBlockEnd + 2; formatedLine.blockState = FGTextBuffer.BlockState.None; } break; case FGTextBuffer.BlockState.StringBlock: int i = startAt; int closingQuote = line.IndexOf('\"', startAt); while (closingQuote != -1 && closingQuote < length - 2 && (line[closingQuote + 1] != '\"' || line[closingQuote + 2] != '\"')) { i = closingQuote + 2; closingQuote = line.IndexOf('\"', i); } if (closingQuote == -1) { tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt)) { formatedLine = formatedLine }); startAt = length; } else { tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt)) { formatedLine = formatedLine }); startAt = closingQuote; tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 3)) { formatedLine = formatedLine }); startAt += 3; formatedLine.blockState = FGTextBuffer.BlockState.None; } break; } } }
public virtual void LexLine(int currentLine, FormatedLine formatedLine) { formatedLine.index = currentLine; if (parserThread != null) parserThread.Join(); parserThread = null; string textLine = textBuffer.lines[currentLine]; var lineTokens = new List<SyntaxToken>(); if (textLine.Length == 0) { formatedLine.tokens = lineTokens; } else { //Tokenize(lineTokens, textLine, ref formatedLine.blockState); lineTokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, textLine) { style = textBuffer.styles.normalStyle, formatedLine = formatedLine }); formatedLine.tokens = lineTokens; var lineWidth = textBuffer.CharIndexToColumn(textLine.Length, currentLine); if (lineWidth > textBuffer.longestLine) textBuffer.longestLine = lineWidth; } }
public virtual void CutParseTree(int fromLine, FormatedLine[] formatedLines) { if (parseTree == null) return; ParseTree.BaseNode cut = null; var prevLine = fromLine; while (cut == null && prevLine --> 0) { var tokens = textBuffer.formatedLines[prevLine].tokens; for (var i = tokens.Count; i --> 0; ) if (tokens[i].tokenKind > SyntaxToken.Kind.LastWSToken && tokens[i].parent != null && tokens[i].parent.syntaxError == null) { cut = tokens[i].parent; break; } } var cutThis = false; if (cut == null) { cut = parseTree.root.ChildAt(0); cutThis = true; } while (cut != null) { var cutParent = cut.parent; if (cutParent == null) break; var cutIndex = cutThis ? cut.childIndex : cut.childIndex + 1; while (cutIndex > 0) { var child = cutParent.ChildAt(cutIndex - 1); if (child != null && !child.HasLeafs()) --cutIndex; else break; } cutThis = cutThis && cutIndex == 0; if (cutIndex < cutParent.numValidNodes) { cutParent.InvalidateFrom(cutIndex); } cut = cutParent; cut.syntaxError = null; } }