public TokenizedScript TokenizeScript(string scriptToProcess)
{
TokenizedScript output = new TokenizedScript();
int lineNumber = 1;
FastString script = new FastString(scriptToProcess);
while (script.Length > 0)
{
SkipWhitespace(ref script);
if (script.Length == 0)
{
break;
}
else if ((script[0] == '\r') || (script[0] == '\n'))
{
output.WriteNewLineNumber(lineNumber);
lineNumber++;
if ((script.Length > 1) && (script[0] == '\r') && (script[1] == '\n'))
{
script = script.Substring(2);
}
else
{
script = script.Substring(1);
}
}
else
{
int symbolStart = 0;
int i = 1;
while (IsPartOfSameSymbol(script, symbolStart, i))
{
i++;
}
string thisSymbol = script.Substring(symbolStart, i);
output.WriteToken(thisSymbol);
if (thisSymbol.StartsWith(Constants.NEW_SCRIPT_MARKER))
{
lineNumber = 1;
}
if (i < script.Length)
{
script = script.Substring(i);
}
else
{
break;
}
}
}
output.WriteEndOfStream();
return output;
}
private static void SkipWhitespace(ref FastString script)
{
while (IsWhitespace(script[0]))
{
script = script.Substring(1);
if (script.Length == 0)
{
break;
}
}
}
private static void SkipWhitespace(ref FastString script)
{
while (IsWhitespace(script[0]))
{
script = script.Substring(1);
if (script.Length == 0)
{
break;
}
}
}
private static bool IncrementIndexToSkipAnyComments(FastString script, ref int index)
{
if (index < script.Length - 1)
{
if ((script[index] == '/') && (script[index + 1] == '/'))
{
while ((script[index] != '\r') && (index < script.Length - 1))
{
index++;
}
}
if ((script[index] == '/') && (script[index + 1] == '*'))
{
index = script.IndexOf("*/", index + 2);
if (index < 0)
{
index = script.Length - 1;
return true;
}
}
}
return false;
}
private static string GetNextWord(ref FastString script)
{
SkipWhitespace(ref script);
if (script.Length == 0)
{
return string.Empty;
}
int index = 0;
while (Char.IsLetterOrDigit(script[index]) || (script[index] == '_'))
{
index++;
if (index >= script.Length)
{
break;
}
}
if (index == 0)
{
index++;
}
if ((script[0] == ':') && (script.Length > 1) && (script[1] == ':'))
{
// Make :: into one word
index++;
}
string nextWord = script.Substring(0, index);
script = script.Substring(index);
return nextWord;
}
private static bool DoesCurrentLineHaveToken(FastString script, string tokenToCheckFor)
{
int indexOfNextLine = script.IndexOf("\r\n");
if (indexOfNextLine > 0)
{
if (script.Substring(0, indexOfNextLine).IndexOf(tokenToCheckFor) > 0)
{
return true;
}
}
return false;
}
private static bool AdjustFunctionListForExtenderFunction(List<ScriptStruct> structs, ref List<ScriptFunction> functionList, ref FastString script)
{
if (script.StartsWith("this "))
{
GetNextWord(ref script);
string structName = GetNextWord(ref script);
while ((script[0] != ',') && (script[0] != ')') && (script.Length > 1))
{
script = script.Substring(1);
}
if (script[0] == ',')
{
script = script.Substring(1);
}
script = script.Trim();
foreach (ScriptStruct struc in structs)
{
if (struc.Name == structName)
{
functionList = struc.Functions;
return true;
}
}
ScriptStruct newStruct = new ScriptStruct(structName);
functionList = newStruct.Functions;
structs.Add(newStruct);
return true;
}
return false;
}
private static void AddVariableDeclaration(List<ScriptVariable> variables, ref FastString script, string thisWord, AutoCompleteParserState state)
{
if ((state.LastWord.Length > 0) && (state.WordBeforeLast.Length > 0))
{
if (!DoesCurrentLineHaveToken(script, AUTO_COMPLETE_IGNORE))
{
bool isArray = false, isPointer = false;
bool isStatic = false, isStaticOnly = false;
bool isNoInherit = false, isProtected = false;
string type = state.WordBeforeLast;
string varName = state.LastWord;
if (thisWord == "[")
{
while ((script.Length > 0) && (GetNextWord(ref script) != "]")) ;
isArray = true;
}
else if (state.DynamicArrayDefinition)
{
varName = state.WordBeforeLast;
type = state.WordBeforeWordBeforeLast;
isArray = true;
}
if (type == "*")
{
isPointer = true;
if (state.DynamicArrayDefinition)
{
type = state.PreviousWords[3];
}
else
{
type = state.WordBeforeWordBeforeLast;
}
}
if (state.IsWordInPreviousList("static"))
{
isStatic = true;
}
if (state.IsWordInPreviousList("protected"))
{
isProtected = true;
}
if (DoesCurrentLineHaveToken(script, AUTO_COMPLETE_STATIC_ONLY))
{
isStaticOnly = true;
}
if (DoesCurrentLineHaveToken(script, AUTO_COMPLETE_NO_INHERIT))
{
isNoInherit = true;
}
// ignore "struct GUI;" prototypes
if (type != "struct")
{
//if (varName == "{") System.Diagnostics.Debugger.Break();
ScriptVariable newVar = new ScriptVariable(varName, type, isArray, isPointer, state.InsideIfDefBlock, state.InsideIfNDefBlock, isStatic, isStaticOnly, isNoInherit, isProtected, state.CurrentScriptCharacterIndex);
if (!string.IsNullOrEmpty(state.PreviousComment))
{
newVar.Description = state.PreviousComment;
state.PreviousComment = null;
}
variables.Add(newVar);
}
}
state.DynamicArrayDefinition = false;
}
}
private string PreProcessLine(string lineToProcess)
{
if (DeletingCurrentLine())
{
return(string.Empty);
}
Stack <StringBuilder> previousOutput = new Stack <StringBuilder>();
Stack <FastString> previousLine = new Stack <FastString>();
StringBuilder output = new StringBuilder(lineToProcess.Length);
FastString line = new FastString(lineToProcess);
Stack <String> ignored = new Stack <String>();
while (line.Length > 0)
{
int i = 0;
while ((i < line.Length) && (!Char.IsLetterOrDigit(line[i])))
{
if ((line[i] == '"') || (line[i] == '\''))
{
i = FindIndexOfMatchingCharacter(line.ToString(), i, line[i]);
if (i < 0)
{
i = line.Length;
break;
}
}
i++;
}
output.Append(line.Substring(0, i));
if (i < line.Length)
{
bool precededByDot = false;
if (i > 0)
{
precededByDot = (line[i - 1] == '.');
}
line = line.Substring(i);
string realStringLine = line.ToString();
string theWord = GetNextWord(ref realStringLine, false, false);
line = realStringLine;
if ((!precededByDot) && (!ignored.Contains(theWord)) && (_state.Macros.Contains(theWord)))
{
previousOutput.Push(output);
previousLine.Push(line);
ignored.Push(theWord);
line = new FastString(_state.Macros[theWord]);
output = new StringBuilder(line.Length);
}
else
{
output.Append(theWord);
}
}
else
{
line = "";
}
while (line.Length == 0 && previousOutput.Count > 0)
{
String result = output.ToString();
output = previousOutput.Pop();
line = previousLine.Pop();
ignored.Pop();
output.Append(result);
}
}
return(output.ToString());
}
private static void SkipWhitespace(ref FastString script)
{
if (script.Length == 0)
{
return;
}
int index = 0;
while ((script[index] == ' ') || (script[index] == '\t')
|| (script[index] == '\r') || (script[index] == '\n'))
{
index++;
if (index >= script.Length)
{
script = string.Empty;
return;
}
}
if (index > 0)
{
script = script.Substring(index);
}
}
private static void ProcessPreProcessorDirective(List<ScriptDefine> defines, ref FastString script, AutoCompleteParserState state)
{
script = script.Substring(1);
string preProcessorDirective = GetNextWord(ref script);
if (preProcessorDirective == "define")
{
string macroName = GetNextWord(ref script);
if ((Char.IsLetter(macroName[0])) &&
(!DoesCurrentLineHaveToken(script, AUTO_COMPLETE_IGNORE)))
{
defines.Add(new ScriptDefine(macroName, state.InsideIfDefBlock, state.InsideIfNDefBlock));
}
}
else if (preProcessorDirective == "undef")
{
string macroName = GetNextWord(ref script);
if (Char.IsLetter(macroName[0]))
{
foreach (ScriptDefine define in defines)
{
if (define.Name == macroName)
{
defines.Remove(define);
break;
}
}
}
}
else if (preProcessorDirective == "ifndef")
{
string macroName = GetNextWord(ref script);
if (Char.IsLetter(macroName[0]))
{
state.InsideIfNDefBlock = macroName;
}
}
else if (preProcessorDirective == "ifdef")
{
string macroName = GetNextWord(ref script);
if (Char.IsLetter(macroName[0]))
{
state.InsideIfDefBlock = macroName;
}
}
else if (preProcessorDirective == "endif")
{
state.InsideIfNDefBlock = null;
state.InsideIfDefBlock = null;
}
GoToNextLine(ref script);
state.ClearPreviousWords();
}
private bool IsPartOfSameSymbol(FastString script, int startIndex, int checkIndex)
{
if (checkIndex >= script.Length)
{
return(false);
}
char startChar = script[startIndex];
char thisChar = script[checkIndex];
if (_endSymbolOnNextCharacter)
{
_endSymbolOnNextCharacter = false;
return(false);
}
if ((startChar == '\"') || (startChar == '\''))
{
if (_nextCharacterIsEscaped)
{
// an escaped " or whatever, so let it through
_nextCharacterIsEscaped = false;
}
else if (thisChar == '\\')
{
_nextCharacterIsEscaped = true;
}
else if (thisChar == startChar)
{
// Force the string to end on the next character
_endSymbolOnNextCharacter = true;
}
return(true);
}
if (Char.IsDigit(startChar))
{
if (Char.IsDigit(thisChar))
{
return(true);
}
// float constant
if (thisChar == '.')
{
return(true);
}
return(false);
}
if (Char.IsLetter(startChar) || (startChar == '_'))
{
return(Char.IsLetterOrDigit(thisChar) || (thisChar == '_'));
}
// ==, >=, <=, !=, etc
if (thisChar == '=')
{
if ((startChar == '=') || (startChar == '<') || (startChar == '>') ||
(startChar == '!') || (startChar == '+') || (startChar == '-'))
{
return(true);
}
}
// && and ||, ++ and --
if ((thisChar == '&') && (startChar == '&'))
{
return(true);
}
if ((thisChar == '|') && (startChar == '|'))
{
return(true);
}
if ((thisChar == '+') && (startChar == '+'))
{
return(true);
}
if ((thisChar == '-') && (startChar == '-'))
{
return(true);
}
// << and >>
if ((thisChar == '<') && (startChar == '<'))
{
return(true);
}
if ((thisChar == '>') && (startChar == '>'))
{
return(true);
}
// ...
if ((thisChar == '.') && (startChar == '.'))
{
return(true);
}
// ::
if ((thisChar == ':') && (startChar == ':'))
{
return(true);
}
return(false);
}
private bool IsPartOfSameSymbol(FastString script, int startIndex, int checkIndex)
{
if (checkIndex >= script.Length)
{
return false;
}
char startChar = script[startIndex];
char thisChar = script[checkIndex];
if (_endSymbolOnNextCharacter)
{
_endSymbolOnNextCharacter = false;
return false;
}
if ((startChar == '\"') || (startChar == '\''))
{
if (_nextCharacterIsEscaped)
{
// an escaped " or whatever, so let it through
_nextCharacterIsEscaped = false;
}
else if (thisChar == '\\')
{
_nextCharacterIsEscaped = true;
}
else if (thisChar == startChar)
{
// Force the string to end on the next character
_endSymbolOnNextCharacter = true;
}
return true;
}
if (Char.IsDigit(startChar))
{
if (Char.IsDigit(thisChar))
{
return true;
}
// float constant
if (thisChar == '.')
{
return true;
}
return false;
}
if (Char.IsLetter(startChar) || (startChar == '_'))
{
return (Char.IsLetterOrDigit(thisChar) || (thisChar == '_'));
}
// ==, >=, <=, !=, etc
if (thisChar == '=')
{
if ((startChar == '=') || (startChar == '<') || (startChar == '>') ||
(startChar == '!') || (startChar == '+') || (startChar == '-'))
{
return true;
}
}
// && and ||, ++ and --
if ((thisChar == '&') && (startChar == '&')) return true;
if ((thisChar == '|') && (startChar == '|')) return true;
if ((thisChar == '+') && (startChar == '+')) return true;
if ((thisChar == '-') && (startChar == '-')) return true;
// << and >>
if ((thisChar == '<') && (startChar == '<')) return true;
if ((thisChar == '>') && (startChar == '>')) return true;
// ...
if ((thisChar == '.') && (startChar == '.')) return true;
// ::
if ((thisChar == ':') && (startChar == ':')) return true;
return false;
}
private static void GoToNextLine(ref FastString script)
{
int indexOfNextLine = script.IndexOf("\r\n");
if (indexOfNextLine < 0)
{
script = string.Empty;
}
else
{
script = script.Substring(indexOfNextLine + 2);
}
}
private string PreProcessLine(string lineToProcess)
{
if (DeletingCurrentLine())
{
return string.Empty;
}
Stack<StringBuilder> previousOutput = new Stack<StringBuilder>();
Stack<FastString> previousLine = new Stack<FastString>();
StringBuilder output = new StringBuilder(lineToProcess.Length);
FastString line = new FastString(lineToProcess);
Stack<String> ignored = new Stack<String>();
while (line.Length > 0)
{
int i = 0;
while ((i < line.Length) && (!Char.IsLetterOrDigit(line[i])))
{
if ((line[i] == '"') || (line[i] == '\''))
{
i = FindIndexOfMatchingCharacter(line.ToString(), i, line[i]);
if (i < 0)
{
i = line.Length;
break;
}
}
i++;
}
output.Append(line.Substring(0, i));
if (i < line.Length)
{
bool precededByDot = false;
if (i > 0) precededByDot = (line[i - 1] == '.');
line = line.Substring(i);
string realStringLine = line.ToString();
string theWord = GetNextWord(ref realStringLine, false, false);
line = realStringLine;
if ((!precededByDot) && (!ignored.Contains(theWord)) && (_state.Macros.Contains(theWord)))
{
previousOutput.Push(output);
previousLine.Push(line);
ignored.Push(theWord);
line = new FastString(_state.Macros[theWord]);
output = new StringBuilder(line.Length);
}
else
{
output.Append(theWord);
}
}
else
line = "";
while (line.Length == 0 && previousOutput.Count > 0)
{
String result = output.ToString();
output = previousOutput.Pop();
line = previousLine.Pop();
ignored.Pop();
output.Append(result);
}
}
return output.ToString();
}
private static string PeekNextWord(FastString script)
{
FastString tester = script;
return GetNextWord(ref tester);
}
private static void AddEnumValue(ScriptEnum insideEnumDefinition, FastString script, string lastWord)
{
if ((lastWord.Length > 0) && (Char.IsLetter(lastWord[0])))
{
if (!DoesCurrentLineHaveToken(script, AUTO_COMPLETE_IGNORE))
{
insideEnumDefinition.EnumValues.Add(lastWord);
}
}
}
private static void SkipUntilMatchingClosingBrace(ref FastString script)
{
int braceIndent = 1, index = 1;
while ((braceIndent > 0) && (index < script.Length))
{
if (IncrementIndexToSkipAnyComments(script, ref index))
{
break;
}
if ((script[index] == '"') || (script[index] == '\''))
{
char firstChar = script[index];
index++;
while (index < script.Length - 1)
{
if ((script[index] == firstChar) &&
((script[index - 1] != '\\') || (script[index - 2] == '\\')))
{
break;
}
index++;
}
}
if (script[index] == '{')
{
braceIndent++;
}
else if (script[index] == '}')
{
braceIndent--;
}
index++;
}
script = script.Substring(index);
}
private static bool AddFunctionDeclaration(List<ScriptFunction> functions, ref FastString script, string thisWord, AutoCompleteParserState state, bool isExtenderMethod)
{
bool succeeded = false;
if ((state.LastWord.Length > 0) && (state.WordBeforeLast.Length > 0))
{
if (!DoesCurrentLineHaveToken(script, AUTO_COMPLETE_IGNORE))
{
string functionName = state.LastWord;
string type = state.WordBeforeLast;
bool isPointer = false, isNoInherit = false;
bool isStatic = false, isStaticOnly = false;
bool isProtected = false;
if (type == "::")
{
functionName = state.WordBeforeWordBeforeLast + "::" + functionName;
type = (state.PreviousWords.Length > 3) ? state.PreviousWords[3] : "unknown";
}
if (type == "*")
{
isPointer = true;
type = state.WordBeforeWordBeforeLast;
}
if (state.DynamicArrayDefinition)
{
// get the type name and the []
type = state.WordBeforeWordBeforeLast + state.WordBeforeLast;
}
if (state.IsWordInPreviousList("static"))
{
isStatic = true;
}
if (state.IsWordInPreviousList("protected"))
{
isProtected = true;
}
if (DoesCurrentLineHaveToken(script, AUTO_COMPLETE_STATIC_ONLY))
{
isStaticOnly = true;
}
if (DoesCurrentLineHaveToken(script, AUTO_COMPLETE_NO_INHERIT))
{
isNoInherit = true;
}
int parameterListEndIndex = script.IndexOf(')');
if (parameterListEndIndex >= 0)
{
string parameterList = script.Substring(0, parameterListEndIndex);
script = script.Substring(parameterListEndIndex + 1);
ScriptFunction newFunc = new ScriptFunction(functionName, type, parameterList, state.InsideIfDefBlock, state.InsideIfNDefBlock, isPointer, isStatic, isStaticOnly, isNoInherit, isProtected, isExtenderMethod, state.CurrentScriptCharacterIndex - 1);
if (!string.IsNullOrEmpty(state.PreviousComment))
{
newFunc.Description = state.PreviousComment;
state.PreviousComment = null;
}
functions.Add(newFunc);
succeeded = true;
}
state.DynamicArrayDefinition = false;
}
}
return succeeded;
}
private string PreProcessLine(string lineToProcess)
{
if (DeletingCurrentLine())
{
return string.Empty;
}
StringBuilder output = new StringBuilder(lineToProcess.Length);
FastString line = new FastString(lineToProcess);
while (line.Length > 0)
{
int i = 0;
while ((i < line.Length) && (!Char.IsLetterOrDigit(line[i])))
{
if ((line[i] == '"') || (line[i] == '\''))
{
i = FindIndexOfMatchingCharacter(line.ToString(), i, line[i]);
if (i < 0)
{
i = line.Length;
break;
}
}
i++;
}
output.Append(line.Substring(0, i));
if (i >= line.Length)
{
break;
}
bool precededByDot = false;
if (i > 0) precededByDot = (line[i - 1] == '.');
line = line.Substring(i);
string realStringLine = line.ToString();
string theWord = GetNextWord(ref realStringLine, false, false);
line = realStringLine;
if ((!precededByDot) && (_state.Macros.Contains(theWord)))
{
output.Append(_state.Macros[theWord]);
}
else
{
output.Append(theWord);
}
}
return output.ToString();
}
private static bool IncrementIndexToSkipAnyComments(FastString script, ref int index)
{
while (index < script.Length - 1 && (script[index] == '/'))
{
if ((script[index + 1] == '/'))
{
index = script.IndexOf('\r', index + 2);
if (index < 0)
{
index = script.Length;
}
}
else
{
if ((script[index + 1] == '*'))
{
index = script.IndexOf("*/", index + 2);
if (index < 0)
{
index = script.Length;
}
}
else
{
break;
}
}
}
return index == script.Length;
}
