public virtual void LexLine(int currentLine, FormatedLine formatedLine)
{
formatedLine.index = currentLine;
if (parserThread != null)
{
parserThread.Join();
}
parserThread = null;
string textLine = textBuffer.lines[currentLine];
var lineTokens = formatedLine.tokens ?? new List <SyntaxToken>();
lineTokens.Clear();
formatedLine.tokens = lineTokens;
if (!string.IsNullOrEmpty(textLine))
{
//Tokenize(lineTokens, textLine, ref formatedLine.blockState);
lineTokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, textLine)
{
style = textBuffer.styles.normalStyle, formatedLine = formatedLine
});
var lineWidth = textBuffer.CharIndexToColumn(textLine.Length, currentLine);
if (lineWidth > textBuffer.longestLine)
{
textBuffer.longestLine = lineWidth;
}
}
}
protected bool ParsePPSymbol(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt)
{
var word = FGParser.ScanIdentifierOrKeyword(line, ref startAt);
if (word == null)
{
return(true);
}
word.tokenKind = SyntaxToken.Kind.PreprocessorSymbol;
formatedLine.tokens.Add(word);
word.formatedLine = formatedLine;
if (word.text == "true")
{
return(true);
}
if (word.text == "false")
{
return(false);
}
if (scriptDefines == null)
{
scriptDefines = new HashSet <string>(UnityEditor.EditorUserBuildSettings.activeScriptCompilationDefines);
}
var isDefined = scriptDefines.Contains(word.text);
return(isDefined);
}
protected bool ParsePPPrimaryExpression(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt)
{
if (line[startAt] == '(')
{
formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, "(")
{
formatedLine = formatedLine
});
++startAt;
var ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
var result = ParsePPOrExpression(line, formatedLine, ref startAt);
if (startAt >= line.Length)
{
//TODO: Insert missing token
return(result);
}
if (line[startAt] == ')')
{
formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, ")")
{
formatedLine = formatedLine
});
++startAt;
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
return(result);
}
//TODO: Insert missing token
return(result);
}
var symbolResult = ParsePPSymbol(line, formatedLine, ref startAt);
var ws2 = ScanWhitespace(line, ref startAt);
if (ws2 != null)
{
formatedLine.tokens.Add(ws2);
ws2.formatedLine = formatedLine;
}
return(symbolResult);
}
protected void OpenRegion(FGTextBuffer.FormatedLine formatedLine, FGTextBuffer.RegionTree.Kind regionKind)
{
var parentRegion = formatedLine.regionTree;
FGTextBuffer.RegionTree reuseRegion = null;
switch (regionKind)
{
case FGTextBuffer.RegionTree.Kind.Else:
case FGTextBuffer.RegionTree.Kind.Elif:
case FGTextBuffer.RegionTree.Kind.InactiveElse:
case FGTextBuffer.RegionTree.Kind.InactiveElif:
parentRegion = parentRegion.parent;
break;
}
if (parentRegion.children != null)
{
for (var i = parentRegion.children.Count; i-- > 0;)
{
if (parentRegion.children[i].line == formatedLine)
{
reuseRegion = parentRegion.children[i];
break;
}
}
}
if (reuseRegion != null)
{
if (reuseRegion.kind == regionKind)
{
formatedLine.regionTree = reuseRegion;
return;
}
reuseRegion.parent = null;
parentRegion.children.Remove(reuseRegion);
}
formatedLine.regionTree = new FGTextBuffer.RegionTree {
parent = parentRegion,
kind = regionKind,
line = formatedLine,
};
if (parentRegion.children == null)
{
parentRegion.children = new List <FGTextBuffer.RegionTree>();
}
parentRegion.children.Add(formatedLine.regionTree);
}
protected bool ParsePPEqualityExpression(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt)
{
if (startAt >= line.Length)
{
//TODO: Insert missing token
return(true);
}
var lhs = ParsePPUnaryExpression(line, formatedLine, ref startAt);
var ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (startAt + 1 < line.Length && (line[startAt] == '=' || line[startAt + 1] == '!') && line[startAt + 1] == '=')
{
var equality = line[startAt] == '=';
formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, equality ? "==" : "!=")
{
formatedLine = formatedLine
});
startAt += 2;
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
var rhs = ParsePPEqualityExpression(line, formatedLine, ref startAt);
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
return(equality ? lhs == rhs : lhs != rhs);
}
return(lhs);
}
protected bool ParsePPAndExpression(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt)
{
if (startAt >= line.Length)
{
//TODO: Insert missing token
return(true);
}
var lhs = ParsePPEqualityExpression(line, formatedLine, ref startAt);
var ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (startAt + 1 < line.Length && line[startAt] == '&' && line[startAt + 1] == '&')
{
formatedLine.tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, "&&")
{
formatedLine = formatedLine
});
startAt += 2;
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
var rhs = ParsePPAndExpression(line, formatedLine, ref startAt);
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
formatedLine.tokens.Add(ws);
ws.formatedLine = formatedLine;
}
return(lhs && rhs);
}
return(lhs);
}
protected virtual void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine)
{
}
protected void CloseRegion(FGTextBuffer.FormatedLine formatedLine)
{
formatedLine.regionTree = formatedLine.regionTree.parent;
}
public override void LexLine(int currentLine, FGTextBuffer.FormatedLine formatedLine)
{
formatedLine.index = currentLine;
if (parserThread != null)
{
parserThread.Join();
}
parserThread = null;
string textLine = textBuffer.lines[currentLine];
//Stopwatch sw1 = new Stopwatch();
//Stopwatch sw2 = new Stopwatch();
if (currentLine == 0)
{
var defaultScriptDefines = UnityEditor.EditorUserBuildSettings.activeScriptCompilationDefines;
if (scriptDefines == null || !scriptDefines.SetEquals(defaultScriptDefines))
{
if (scriptDefines == null)
{
scriptDefines = new HashSet <string>(defaultScriptDefines);
}
else
{
scriptDefines.Clear();
scriptDefines.UnionWith(defaultScriptDefines);
}
}
}
//sw2.Start();
Tokenize(textLine, formatedLine);
// syntaxTree.SetLineTokens(currentLine, lineTokens);
var lineTokens = formatedLine.tokens;
if (textLine.Length == 0)
{
formatedLine.tokens.Clear();
}
else if (textBuffer.styles != null)
{
var lineWidth = textBuffer.CharIndexToColumn(textLine.Length, currentLine);
if (lineWidth > textBuffer.longestLine)
{
textBuffer.longestLine = lineWidth;
}
for (var i = 0; i < lineTokens.Count; ++i)
{
var token = lineTokens[i];
switch (token.tokenKind)
{
case SyntaxToken.Kind.Whitespace:
case SyntaxToken.Kind.Missing:
token.style = textBuffer.styles.normalStyle;
break;
case SyntaxToken.Kind.Punctuator:
token.style = IsOperator(token.text) ? textBuffer.styles.operatorStyle : textBuffer.styles.punctuatorStyle;
break;
case SyntaxToken.Kind.Keyword:
if (IsBuiltInType(token.text))
{
if (token.text == "string" || token.text == "object")
{
token.style = textBuffer.styles.builtInRefTypeStyle;
}
else
{
token.style = textBuffer.styles.builtInValueTypeStyle;
}
}
else
{
token.style = textBuffer.styles.keywordStyle;
}
break;
case SyntaxToken.Kind.Identifier:
if (IsBuiltInLiteral(token.text))
{
token.style = textBuffer.styles.builtInLiteralsStyle;
token.tokenKind = SyntaxToken.Kind.BuiltInLiteral;
}
else if (IsUnityType(token.text))
{
token.style = textBuffer.styles.referenceTypeStyle;
}
else
{
token.style = textBuffer.styles.normalStyle;
}
break;
case SyntaxToken.Kind.IntegerLiteral:
case SyntaxToken.Kind.RealLiteral:
token.style = textBuffer.styles.constantStyle;
break;
case SyntaxToken.Kind.Comment:
var regionKind = formatedLine.regionTree.kind;
var inactiveLine = regionKind > FGTextBuffer.RegionTree.Kind.LastActive;
token.style = inactiveLine ? textBuffer.styles.inactiveCodeStyle : textBuffer.styles.commentStyle;
break;
case SyntaxToken.Kind.Preprocessor:
token.style = textBuffer.styles.preprocessorStyle;
break;
case SyntaxToken.Kind.PreprocessorSymbol:
token.style = textBuffer.styles.defineSymbols;
break;
case SyntaxToken.Kind.PreprocessorArguments:
case SyntaxToken.Kind.PreprocessorCommentExpected:
case SyntaxToken.Kind.PreprocessorDirectiveExpected:
case SyntaxToken.Kind.PreprocessorUnexpectedDirective:
token.style = textBuffer.styles.normalStyle;
break;
case SyntaxToken.Kind.CharLiteral:
case SyntaxToken.Kind.StringLiteral:
case SyntaxToken.Kind.VerbatimStringBegin:
case SyntaxToken.Kind.VerbatimStringLiteral:
token.style = textBuffer.styles.stringStyle;
break;
}
lineTokens[i] = token;
}
}
}
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine)
{
var tokens = new List <SyntaxToken>();
formatedLine.tokens = tokens;
int startAt = 0;
int length = line.Length;
SyntaxToken token;
SyntaxToken ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (formatedLine.blockState == FGTextBuffer.BlockState.None && startAt < length && line[startAt] == '#')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Preprocessor, "#")
{
formatedLine = formatedLine
});
++startAt;
//ws = ScanWhitespace(line, ref startAt);
//if (ws != null)
//{
// tokens.Add(ws);
// ws.formatedLine = formatedLine;
//}
var error = false;
var commentsOnly = false;
var preprocessorCommentsAllowed = true;
token = ScanWord(line, ref startAt);
if (Array.BinarySearch(preprocessorKeywords, token.text) < 0)
{
token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected;
tokens.Add(token);
token.formatedLine = formatedLine;
error = true;
}
else
{
token.tokenKind = SyntaxToken.Kind.Preprocessor;
tokens.Add(token);
token.formatedLine = formatedLine;
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (token.text == "if")
{
bool active = ParsePPOrExpression(line, formatedLine, ref startAt);
bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (active && !inactiveParent)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.If);
commentsOnly = true;
}
else
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveIf);
commentsOnly = true;
}
}
else if (token.text == "elif")
{
bool active = ParsePPOrExpression(line, formatedLine, ref startAt);
bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif);
}
else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf)
{
inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (active && !inactiveParent)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Elif);
}
else
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif);
}
}
else
{
token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
}
}
else if (token.text == "else")
{
if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse);
}
else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif)
{
bool inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (inactiveParent)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse);
}
else
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Else);
}
}
else
{
token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
}
}
else if (token.text == "endif")
{
if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Else ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElse)
{
CloseRegion(formatedLine);
}
else
{
token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
}
}
//else if (token.text == "region")
//{
// var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
// if (inactive)
// {
// OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveRegion);
// }
// else
// {
// OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Region);
// }
// preprocessorCommentsAllowed = false;
//}
//else if (token.text == "endregion")
//{
// if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Region ||
// formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveRegion)
// {
// CloseRegion(formatedLine);
// }
// else
// {
// token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
// }
// preprocessorCommentsAllowed = false;
//}
//else if (token.text == "define" || token.text == "undef")
//{
// var symbol = FGParser.ScanIdentifierOrKeyword(line, ref startAt);
// if (symbol != null && symbol.text != "true" && symbol.text != "false")
// {
// symbol.tokenKind = SyntaxToken.Kind.PreprocessorSymbol;
// formatedLine.tokens.Add(symbol);
// symbol.formatedLine = formatedLine;
// scriptDefinesChanged = true;
// var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
// if (!inactive)
// {
// if (token.text == "define")
// {
// if (!scriptDefines.Contains(symbol.text))
// {
// scriptDefines.Add(symbol.text);
// //scriptDefinesChanged = true;
// }
// }
// else
// {
// if (scriptDefines.Contains(symbol.text))
// {
// scriptDefines.Remove(symbol.text);
// //scriptDefinesChanged = true;
// }
// }
// }
// }
//}
//else if (token.text == "error" || token.text == "warning")
//{
// preprocessorCommentsAllowed = false;
//}
}
if (!preprocessorCommentsAllowed)
{
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (startAt < length)
{
var textArgument = line.Substring(startAt);
textArgument.TrimEnd(new [] { ' ', '\t' });
tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, textArgument)
{
formatedLine = formatedLine
});
startAt = length - textArgument.Length;
if (startAt < length)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Whitespace, line.Substring(startAt))
{
formatedLine = formatedLine
});
}
}
return;
}
while (startAt < length)
{
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
continue;
}
var firstChar = line[startAt];
if (startAt < length - 1 && firstChar == '/' && line[startAt + 1] == '/')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
break;
}
else if (commentsOnly)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorCommentExpected, line.Substring(startAt))
{
formatedLine = formatedLine
});
break;
}
if (char.IsLetterOrDigit(firstChar) || firstChar == '_')
{
token = ScanWord(line, ref startAt);
token.tokenKind = SyntaxToken.Kind.PreprocessorArguments;
tokens.Add(token);
token.formatedLine = formatedLine;
}
else if (firstChar == '"')
{
token = ScanStringLiteral(line, ref startAt);
token.tokenKind = SyntaxToken.Kind.PreprocessorArguments;
tokens.Add(token);
token.formatedLine = formatedLine;
}
else if (firstChar == '\'')
{
token = ScanCharLiteral(line, ref startAt);
token.tokenKind = SyntaxToken.Kind.PreprocessorArguments;
tokens.Add(token);
token.formatedLine = formatedLine;
}
else
{
token = new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, firstChar.ToString())
{
formatedLine = formatedLine
};
tokens.Add(token);
++startAt;
}
if (error)
{
token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected;
}
}
return;
}
var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
while (startAt < length)
{
switch (formatedLine.blockState)
{
case FGTextBuffer.BlockState.None:
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
continue;
}
if (inactiveLine)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
if (line[startAt] == '/' && startAt < length - 1)
{
if (line[startAt + 1] == '/')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "//")
{
formatedLine = formatedLine
});
startAt += 2;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
else if (line[startAt + 1] == '*')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "/*")
{
formatedLine = formatedLine
});
startAt += 2;
formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock;
break;
}
}
if (line[startAt] == '\'')
{
token = ScanCharLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
if (line[startAt] == '\"')
{
token = ScanStringLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
//if (startAt < length - 1 && line[startAt] == '@' && line[startAt + 1] == '\"')
//{
// token = new SyntaxToken(SyntaxToken.Kind.VerbatimStringBegin, line.Substring(startAt, 2)) { formatedLine = formatedLine };
// tokens.Add(token);
// startAt += 2;
// formatedLine.blockState = FGTextBuffer.BlockState.StringBlock;
// break;
//}
if (line[startAt] >= '0' && line[startAt] <= '9' ||
startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9')
{
token = ScanNumericLiteral_JS(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
token = ScanIdentifierOrKeyword(line, ref startAt);
if (token != null)
{
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
// Multi-character operators / punctuators
// "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "??", "+=", "-=", "*=", "/=", "%=",
// "&=", "|=", "^=", "<<=", ">>=", "=>", "::", "->", "^="
var punctuatorStart = startAt++;
if (startAt < line.Length)
{
switch (line[punctuatorStart])
{
case '?':
if (line[startAt] == '?')
{
++startAt;
}
break;
case '+':
if (line[startAt] == '+' || line[startAt] == '=' || line[startAt] == '>')
{
++startAt;
}
break;
case '-':
if (line[startAt] == '-' || line[startAt] == '=')
{
++startAt;
}
break;
case '<':
if (line[startAt] == '=')
{
++startAt;
}
else if (line[startAt] == '<')
{
++startAt;
if (startAt < line.Length && line[startAt] == '=')
{
++startAt;
}
}
break;
case '>':
if (line[startAt] == '=')
{
++startAt;
}
else if (startAt < line.Length && line[startAt] == '>')
{
++startAt;
if (line[startAt] == '=')
{
++startAt;
}
}
break;
case '=':
if (line[startAt] == '=' || line[startAt] == '>')
{
++startAt;
}
break;
case '&':
if (line[startAt] == '=' || line[startAt] == '&')
{
++startAt;
}
break;
case '|':
if (line[startAt] == '=' || line[startAt] == '|')
{
++startAt;
}
break;
case '*':
case '/':
case '%':
case '^':
case '!':
if (line[startAt] == '=')
{
++startAt;
}
break;
case ':':
if (line[startAt] == ':')
{
++startAt;
}
break;
}
}
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart))
{
formatedLine = formatedLine
});
break;
case FGTextBuffer.BlockState.CommentBlock:
int commentBlockEnd = line.IndexOf("*/", startAt, StringComparison.Ordinal);
if (commentBlockEnd == -1)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
}
else
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt))
{
formatedLine = formatedLine
});
startAt = commentBlockEnd + 2;
formatedLine.blockState = FGTextBuffer.BlockState.None;
}
break;
//case FGTextBuffer.BlockState.StringBlock:
// int i = startAt;
// int closingQuote = line.IndexOf('\"', startAt);
// while (closingQuote != -1 && closingQuote < length - 1 && line[closingQuote + 1] == '\"')
// {
// i = closingQuote + 2;
// closingQuote = line.IndexOf('\"', i);
// }
// if (closingQuote == -1)
// {
// tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt)) { formatedLine = formatedLine });
// startAt = length;
// }
// else
// {
// tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt)) { formatedLine = formatedLine });
// startAt = closingQuote;
// tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 1)) { formatedLine = formatedLine });
// ++startAt;
// formatedLine.blockState = FGTextBuffer.BlockState.None;
// }
// break;
}
}
}
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine)
{
var tokens = formatedLine.tokens ?? new List <SyntaxToken>();
formatedLine.tokens = tokens;
tokens.Clear();
int startAt = 0;
int length = line.Length;
SyntaxToken token;
SyntaxToken ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
while (startAt < length)
{
switch (formatedLine.blockState)
{
case FGTextBuffer.BlockState.None:
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
continue;
}
if (inactiveLine)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
//注释--、注释块开始--[[
if (line[startAt] == '-' && startAt < length - 1)
{
if (startAt + 3 <= length - 1 && line.Substring(startAt, 4) == "--[[")
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "--[[")
{
formatedLine = formatedLine
});
startAt += 4;
formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock;
break;
}
else if (line[startAt + 1] == '-')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "--")
{
formatedLine = formatedLine
});
startAt += 2;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
}
if (line[startAt] == '\'')
{
token = ScanCharLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
if (line[startAt] == '\"')
{
token = ScanStringLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
if (line[startAt] >= '0' && line[startAt] <= '9' ||
startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9')
{
token = ScanNumericLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
token = ScanIdentifierOrKeyword(line, ref startAt);
if (token != null)
{
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
var punctuatorStart = startAt++;
if (startAt < line.Length)
{
switch (line[punctuatorStart])
{
case '?':
if (line[startAt] == '?')
{
++startAt;
}
break;
case '+':
if (line[startAt] == '+' || line[startAt] == '=')
{
++startAt;
}
break;
case '-':
if (line[startAt] == '-' || line[startAt] == '=')
{
++startAt;
}
break;
case '<':
if (line[startAt] == '=')
{
++startAt;
}
else if (line[startAt] == '<')
{
++startAt;
if (startAt < line.Length && line[startAt] == '=')
{
++startAt;
}
}
break;
case '>':
if (line[startAt] == '=')
{
++startAt;
}
//else if (startAt < line.Length && line[startAt] == '>')
//{
// ++startAt;
// if (line[startAt] == '=')
// ++startAt;
//}
break;
case '=':
if (line[startAt] == '=' || line[startAt] == '>')
{
++startAt;
}
break;
case '&':
if (line[startAt] == '=' || line[startAt] == '&')
{
++startAt;
}
break;
case '|':
if (line[startAt] == '=' || line[startAt] == '|')
{
++startAt;
}
break;
case '*':
case '/':
case '%':
case '^':
case '!':
if (line[startAt] == '=')
{
++startAt;
}
break;
case ':':
if (line[startAt] == ':')
{
++startAt;
}
break;
}
}
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart))
{
formatedLine = formatedLine
});
break;
case FGTextBuffer.BlockState.CommentBlock: //注释
int commentBlockEnd = line.IndexOf("]]", startAt, StringComparison.Ordinal);
if (commentBlockEnd == -1)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
}
else
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt))
{
formatedLine = formatedLine
});
startAt = commentBlockEnd + 2;
formatedLine.blockState = FGTextBuffer.BlockState.None;
}
break;
case FGTextBuffer.BlockState.StringBlock:
int i = startAt;
int closingQuote = line.IndexOf('\"', startAt);
while (closingQuote != -1 && closingQuote < length - 1 && line[closingQuote + 1] == '\"')
{
i = closingQuote + 2;
closingQuote = line.IndexOf('\"', i);
}
if (closingQuote == -1)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
}
else
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt))
{
formatedLine = formatedLine
});
startAt = closingQuote;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 1))
{
formatedLine = formatedLine
});
++startAt;
formatedLine.blockState = FGTextBuffer.BlockState.None;
}
break;
}
}
}
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine)
{
var tokens = new List <SyntaxToken>();
formatedLine.tokens = tokens;
int startAt = 0;
int length = line.Length;
SyntaxToken token;
SyntaxToken ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
while (startAt < length)
{
switch (formatedLine.blockState)
{
case FGTextBuffer.BlockState.None:
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
continue;
}
if (inactiveLine)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
if (line[startAt] == '#')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "#")
{
formatedLine = formatedLine
});
++startAt;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
if (line[startAt] == '/' && startAt < length - 1)
{
if (line[startAt + 1] == '/')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "//")
{
formatedLine = formatedLine
});
startAt += 2;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
else if (line[startAt + 1] == '*')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "/*")
{
formatedLine = formatedLine
});
startAt += 2;
formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock;
break;
}
}
if (line[startAt] == '\'')
{
token = ScanCharLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
if (line[startAt] == '\"')
{
if (startAt < length - 2 && line[startAt + 1] == '\"' && line[startAt + 2] == '\"')
{
token = new SyntaxToken(SyntaxToken.Kind.VerbatimStringBegin, line.Substring(startAt, 3))
{
formatedLine = formatedLine
};
tokens.Add(token);
startAt += 3;
formatedLine.blockState = FGTextBuffer.BlockState.StringBlock;
break;
}
token = ScanStringLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
if (line[startAt] >= '0' && line[startAt] <= '9' ||
startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9')
{
token = ScanNumericLiteral_JS(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
token = ScanIdentifierOrKeyword(line, ref startAt);
if (token != null)
{
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
// Multi-character operators / punctuators
// "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "??", "+=", "-=", "*=", "/=", "%=",
// "&=", "|=", "^=", "<<=", ">>=", "=>", "::", "->", "^="
var punctuatorStart = startAt++;
if (startAt < line.Length)
{
switch (line[punctuatorStart])
{
case '?':
if (line[startAt] == '?')
{
++startAt;
}
break;
case '+':
if (line[startAt] == '+' || line[startAt] == '=' || line[startAt] == '>')
{
++startAt;
}
break;
case '-':
if (line[startAt] == '-' || line[startAt] == '=')
{
++startAt;
}
break;
case '<':
if (line[startAt] == '=')
{
++startAt;
}
else if (line[startAt] == '<')
{
++startAt;
if (startAt < line.Length && line[startAt] == '=')
{
++startAt;
}
}
break;
case '>':
if (line[startAt] == '=')
{
++startAt;
}
else if (startAt < line.Length && line[startAt] == '>')
{
++startAt;
if (line[startAt] == '=')
{
++startAt;
}
}
break;
case '=':
if (line[startAt] == '=' || line[startAt] == '>')
{
++startAt;
}
break;
case '&':
if (line[startAt] == '=' || line[startAt] == '&')
{
++startAt;
}
break;
case '|':
if (line[startAt] == '=' || line[startAt] == '|')
{
++startAt;
}
break;
case '*':
case '/':
case '%':
case '^':
case '!':
if (line[startAt] == '=')
{
++startAt;
}
break;
case ':':
if (line[startAt] == ':')
{
++startAt;
}
break;
}
}
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart))
{
formatedLine = formatedLine
});
break;
case FGTextBuffer.BlockState.CommentBlock:
int commentBlockEnd = line.IndexOf("*/", startAt, StringComparison.Ordinal);
if (commentBlockEnd == -1)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
}
else
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt))
{
formatedLine = formatedLine
});
startAt = commentBlockEnd + 2;
formatedLine.blockState = FGTextBuffer.BlockState.None;
}
break;
case FGTextBuffer.BlockState.StringBlock:
int i = startAt;
int closingQuote = line.IndexOf('\"', startAt);
while (closingQuote != -1 && closingQuote < length - 2 &&
(line[closingQuote + 1] != '\"' || line[closingQuote + 2] != '\"'))
{
i = closingQuote + 2;
closingQuote = line.IndexOf('\"', i);
}
if (closingQuote == -1)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
}
else
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt))
{
formatedLine = formatedLine
});
startAt = closingQuote;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 3))
{
formatedLine = formatedLine
});
startAt += 3;
formatedLine.blockState = FGTextBuffer.BlockState.None;
}
break;
}
}
}
public virtual void LexLine(int currentLine, FormatedLine formatedLine)
{
formatedLine.index = currentLine;
if (parserThread != null)
parserThread.Join();
parserThread = null;
string textLine = textBuffer.lines[currentLine];
var lineTokens = new List<SyntaxToken>();
if (textLine.Length == 0)
{
formatedLine.tokens = lineTokens;
}
else
{
//Tokenize(lineTokens, textLine, ref formatedLine.blockState);
lineTokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, textLine) { style = textBuffer.styles.normalStyle, formatedLine = formatedLine });
formatedLine.tokens = lineTokens;
var lineWidth = textBuffer.CharIndexToColumn(textLine.Length, currentLine);
if (lineWidth > textBuffer.longestLine)
textBuffer.longestLine = lineWidth;
}
}
public virtual void CutParseTree(int fromLine, FormatedLine[] formatedLines)
{
if (parseTree == null)
return;
ParseTree.BaseNode cut = null;
var prevLine = fromLine;
while (cut == null && prevLine --> 0)
{
var tokens = textBuffer.formatedLines[prevLine].tokens;
for (var i = tokens.Count; i --> 0; )
if (tokens[i].tokenKind > SyntaxToken.Kind.LastWSToken && tokens[i].parent != null && tokens[i].parent.syntaxError == null)
{
cut = tokens[i].parent;
break;
}
}
var cutThis = false;
if (cut == null)
{
cut = parseTree.root.ChildAt(0);
cutThis = true;
}
while (cut != null)
{
var cutParent = cut.parent;
if (cutParent == null)
break;
var cutIndex = cutThis ? cut.childIndex : cut.childIndex + 1;
while (cutIndex > 0)
{
var child = cutParent.ChildAt(cutIndex - 1);
if (child != null && !child.HasLeafs())
--cutIndex;
else
break;
}
cutThis = cutThis && cutIndex == 0;
if (cutIndex < cutParent.numValidNodes)
{
cutParent.InvalidateFrom(cutIndex);
}
cut = cutParent;
cut.syntaxError = null;
}
}
