protected bool ParsePPSymbol(string line, FGTextBuffer.FormatedLine formatedLine, ref int startAt)
{
var word = FGParser.ScanIdentifierOrKeyword(line, ref startAt);
if (word == null)
{
return(true);
}
word.tokenKind = SyntaxToken.Kind.PreprocessorSymbol;
formatedLine.tokens.Add(word);
word.formatedLine = formatedLine;
if (word.text == "true")
{
return(true);
}
if (word.text == "false")
{
return(false);
}
if (scriptDefines == null)
{
scriptDefines = new HashSet <string>(UnityEditor.EditorUserBuildSettings.activeScriptCompilationDefines);
}
var isDefined = scriptDefines.Contains(word.text);
return(isDefined);
}
private new SyntaxToken ScanIdentifierOrKeyword(string line, ref int startAt)
{
var token = FGParser.ScanIdentifierOrKeyword(line, ref startAt);
if (token != null && token.tokenKind == SyntaxToken.Kind.Keyword && !IsKeywordOrBuiltInType(token.text))
{
token.tokenKind = SyntaxToken.Kind.Identifier;
}
return(token);
}
protected override void Tokenize(string line, FGTextBuffer.FormatedLine formatedLine)
{
var tokens = formatedLine.tokens ?? new List <SyntaxToken>();
formatedLine.tokens = tokens;
tokens.Clear();
int startAt = 0;
int length = line.Length;
SyntaxToken token;
SyntaxToken ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (formatedLine.blockState == FGTextBuffer.BlockState.None && startAt < length && line[startAt] == '#')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Preprocessor, "#")
{
formatedLine = formatedLine
});
++startAt;
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
var error = false;
var commentsOnly = false;
var preprocessorCommentsAllowed = true;
token = ScanWord(line, ref startAt);
if (Array.BinarySearch(preprocessorKeywords, token.text) < 0)
{
token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected;
tokens.Add(token);
token.formatedLine = formatedLine;
error = true;
}
else
{
token.tokenKind = SyntaxToken.Kind.Preprocessor;
tokens.Add(token);
token.formatedLine = formatedLine;
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (token.text == "if")
{
bool active = ParsePPOrExpression(line, formatedLine, ref startAt);
bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (active && !inactiveParent)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.If);
commentsOnly = true;
}
else
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveIf);
commentsOnly = true;
}
}
else if (token.text == "elif")
{
bool active = ParsePPOrExpression(line, formatedLine, ref startAt);
bool inactiveParent = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif);
}
else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf)
{
inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (active && !inactiveParent)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Elif);
}
else
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElif);
}
}
else
{
token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
}
}
else if (token.text == "else")
{
if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse);
}
else if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif)
{
bool inactiveParent = formatedLine.regionTree.parent.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (inactiveParent)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveElse);
}
else
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Else);
}
}
else
{
token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
}
}
else if (token.text == "endif")
{
if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.If ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Elif ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Else ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveIf ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElif ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveElse)
{
CloseRegion(formatedLine);
}
else
{
token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
}
}
else if (token.text == "region")
{
var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (inactive)
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.InactiveRegion);
}
else
{
OpenRegion(formatedLine, FGTextBuffer.RegionTree.Kind.Region);
}
preprocessorCommentsAllowed = false;
}
else if (token.text == "endregion")
{
if (formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.Region ||
formatedLine.regionTree.kind == FGTextBuffer.RegionTree.Kind.InactiveRegion)
{
CloseRegion(formatedLine);
}
else
{
token.tokenKind = SyntaxToken.Kind.PreprocessorUnexpectedDirective;
}
preprocessorCommentsAllowed = false;
}
else if (token.text == "define" || token.text == "undef")
{
var symbol = FGParser.ScanIdentifierOrKeyword(line, ref startAt);
if (symbol != null && symbol.text != "true" && symbol.text != "false")
{
symbol.tokenKind = SyntaxToken.Kind.PreprocessorSymbol;
formatedLine.tokens.Add(symbol);
symbol.formatedLine = formatedLine;
scriptDefinesChanged = true;
var inactive = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
if (!inactive)
{
if (token.text == "define")
{
if (!scriptDefines.Contains(symbol.text))
{
scriptDefines.Add(symbol.text);
//scriptDefinesChanged = true;
}
}
else
{
if (scriptDefines.Contains(symbol.text))
{
scriptDefines.Remove(symbol.text);
//scriptDefinesChanged = true;
}
}
}
}
}
else if (token.text == "error" || token.text == "warning")
{
preprocessorCommentsAllowed = false;
}
}
if (!preprocessorCommentsAllowed)
{
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
}
if (startAt < length)
{
var textArgument = line.Substring(startAt);
textArgument.TrimEnd(new [] { ' ', '\t' });
tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, textArgument)
{
formatedLine = formatedLine
});
startAt = length - textArgument.Length;
if (startAt < length)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Whitespace, line.Substring(startAt))
{
formatedLine = formatedLine
});
}
}
return;
}
while (startAt < length)
{
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
continue;
}
var firstChar = line[startAt];
if (startAt < length - 1 && firstChar == '/' && line[startAt + 1] == '/')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
break;
}
else if (commentsOnly)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.PreprocessorCommentExpected, line.Substring(startAt))
{
formatedLine = formatedLine
});
break;
}
if (char.IsLetterOrDigit(firstChar) || firstChar == '_')
{
token = ScanWord(line, ref startAt);
token.tokenKind = SyntaxToken.Kind.PreprocessorArguments;
tokens.Add(token);
token.formatedLine = formatedLine;
}
else if (firstChar == '"')
{
token = ScanStringLiteral(line, ref startAt);
token.tokenKind = SyntaxToken.Kind.PreprocessorArguments;
tokens.Add(token);
token.formatedLine = formatedLine;
}
else if (firstChar == '\'')
{
token = ScanCharLiteral(line, ref startAt);
token.tokenKind = SyntaxToken.Kind.PreprocessorArguments;
tokens.Add(token);
token.formatedLine = formatedLine;
}
else
{
token = new SyntaxToken(SyntaxToken.Kind.PreprocessorArguments, firstChar.ToString())
{
formatedLine = formatedLine
};
tokens.Add(token);
++startAt;
}
if (error)
{
token.tokenKind = SyntaxToken.Kind.PreprocessorDirectiveExpected;
}
}
return;
}
var inactiveLine = formatedLine.regionTree.kind > FGTextBuffer.RegionTree.Kind.LastActive;
while (startAt < length)
{
switch (formatedLine.blockState)
{
case FGTextBuffer.BlockState.None:
ws = ScanWhitespace(line, ref startAt);
if (ws != null)
{
tokens.Add(ws);
ws.formatedLine = formatedLine;
continue;
}
if (inactiveLine)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
if (line[startAt] == '/' && startAt < length - 1)
{
if (line[startAt + 1] == '/')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "//")
{
formatedLine = formatedLine
});
startAt += 2;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
break;
}
else if (line[startAt + 1] == '*')
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, "/*")
{
formatedLine = formatedLine
});
startAt += 2;
formatedLine.blockState = FGTextBuffer.BlockState.CommentBlock;
break;
}
}
if (line[startAt] == '\'')
{
token = ScanCharLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
if (line[startAt] == '\"')
{
token = ScanStringLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
if (startAt < length - 1 && line[startAt] == '@' && line[startAt + 1] == '\"')
{
token = new SyntaxToken(SyntaxToken.Kind.VerbatimStringBegin, line.Substring(startAt, 2))
{
formatedLine = formatedLine
};
tokens.Add(token);
startAt += 2;
formatedLine.blockState = FGTextBuffer.BlockState.StringBlock;
break;
}
if (line[startAt] >= '0' && line[startAt] <= '9' ||
startAt < length - 1 && line[startAt] == '.' && line[startAt + 1] >= '0' && line[startAt + 1] <= '9')
{
token = ScanNumericLiteral(line, ref startAt);
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
token = ScanIdentifierOrKeyword(line, ref startAt);
if (token != null)
{
tokens.Add(token);
token.formatedLine = formatedLine;
break;
}
// Multi-character operators / punctuators
// "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "??", "+=", "-=", "*=", "/=", "%=",
// "&=", "|=", "^=", "<<=", ">>=", "=>", "::"
var punctuatorStart = startAt++;
if (startAt < line.Length)
{
switch (line[punctuatorStart])
{
case '?':
if (line[startAt] == '?')
{
++startAt;
}
break;
case '+':
if (line[startAt] == '+' || line[startAt] == '=')
{
++startAt;
}
break;
case '-':
if (line[startAt] == '-' || line[startAt] == '=')
{
++startAt;
}
break;
case '<':
if (line[startAt] == '=')
{
++startAt;
}
else if (line[startAt] == '<')
{
++startAt;
if (startAt < line.Length && line[startAt] == '=')
{
++startAt;
}
}
break;
case '>':
if (line[startAt] == '=')
{
++startAt;
}
//else if (startAt < line.Length && line[startAt] == '>')
//{
// ++startAt;
// if (line[startAt] == '=')
// ++startAt;
//}
break;
case '=':
if (line[startAt] == '=' || line[startAt] == '>')
{
++startAt;
}
break;
case '&':
if (line[startAt] == '=' || line[startAt] == '&')
{
++startAt;
}
break;
case '|':
if (line[startAt] == '=' || line[startAt] == '|')
{
++startAt;
}
break;
case '*':
case '/':
case '%':
case '^':
case '!':
if (line[startAt] == '=')
{
++startAt;
}
break;
case ':':
if (line[startAt] == ':')
{
++startAt;
}
break;
}
}
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Punctuator, line.Substring(punctuatorStart, startAt - punctuatorStart))
{
formatedLine = formatedLine
});
break;
case FGTextBuffer.BlockState.CommentBlock:
int commentBlockEnd = line.IndexOf("*/", startAt, StringComparison.Ordinal);
if (commentBlockEnd == -1)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
}
else
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.Comment, line.Substring(startAt, commentBlockEnd + 2 - startAt))
{
formatedLine = formatedLine
});
startAt = commentBlockEnd + 2;
formatedLine.blockState = FGTextBuffer.BlockState.None;
}
break;
case FGTextBuffer.BlockState.StringBlock:
int i = startAt;
int closingQuote = line.IndexOf('\"', startAt);
while (closingQuote != -1 && closingQuote < length - 1 && line[closingQuote + 1] == '\"')
{
i = closingQuote + 2;
closingQuote = line.IndexOf('\"', i);
}
if (closingQuote == -1)
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt))
{
formatedLine = formatedLine
});
startAt = length;
}
else
{
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, closingQuote - startAt))
{
formatedLine = formatedLine
});
startAt = closingQuote;
tokens.Add(new SyntaxToken(SyntaxToken.Kind.VerbatimStringLiteral, line.Substring(startAt, 1))
{
formatedLine = formatedLine
});
++startAt;
formatedLine.blockState = FGTextBuffer.BlockState.None;
}
break;
}
}
}
