/// <summary> /// Creates a new AI token. /// </summary> /// <param name="type">The type of this token.</param> /// <param name="content">The string value of this token.</param> /// <param name="startPoint">The starting point of this token.</param> /// <param name="length">The length of this token.</param> public AiToken(AiTokenTypes type, string content, SnapshotPoint startPoint, int length) { // Store parameters Type = type; Content = content; StartPoint = startPoint; Length = length; }
/// <summary> /// Returns the tokens before or after the given position, including all that are in the same line as the starting point. /// This method does not do any context checking, it just scans for known keywords. /// </summary> /// <param name="startingPoint">The position where the parsing shall start.</param> /// <param name="backwards">Optional. Determines whether the token search shall be performed backwards beginning with the given starting point.</param> /// <returns></returns> public IEnumerable <AiToken> GetTokens(SnapshotPoint startingPoint, bool backwards = false) { // Go backwards through lines and return tokens successively ITextSnapshotLine currentLine = startingPoint.GetContainingLine(); string currentLineString = currentLine.GetText(); int currentLineIndex = 0; List <AiToken> lineTokens = new List <AiToken>(); int lastLineNumber = (backwards ? 0 : currentLine.Snapshot.LineCount - 1); while (true) { // Read one line at once, independent of parsing direction lineTokens.Clear(); while (currentLineIndex < currentLine.Length) { // Handle character types char currentLineChar = currentLineString[currentLineIndex]; if (currentLineChar == '(') { lineTokens.Add(new AiToken(AiTokenTypes.OpeningBrace, "(", currentLine.Start + currentLineIndex, 1)); } else if (currentLineChar == ')') { lineTokens.Add(new AiToken(AiTokenTypes.ClosingBrace, ")", currentLine.Start + currentLineIndex, 1)); } else if (currentLineChar == '"') { // Read whole string int nextQuoteIndex = currentLineString.IndexOf('"', currentLineIndex + 1); if (nextQuoteIndex > currentLineIndex) { // Found the closing quotation mark, copy string and update current index int stringLength = nextQuoteIndex - currentLineIndex + 1; lineTokens.Add(new AiToken(AiTokenTypes.String, currentLineString.Substring(currentLineIndex, stringLength), currentLine.Start + currentLineIndex, stringLength)); currentLineIndex = nextQuoteIndex; } else { // The line ends with an unclosed string, copy remainder of the line string remainingLineString = currentLineString.Substring(currentLineIndex); lineTokens.Add(new AiToken(AiTokenTypes.String, remainingLineString, currentLine.Start + currentLineIndex, remainingLineString.Length)); break; } } else if (currentLineChar == ';') { // Copy remainder of the line string remainingLineString = currentLineString.Substring(currentLineIndex); lineTokens.Add(new AiToken(AiTokenTypes.Comment, remainingLineString, currentLine.Start + currentLineIndex, remainingLineString.Length)); break; } else if (currentLineChar == '=') { // This might be the rule arrow, the equality testing operator or a syntax error => inspect next char if (currentLineIndex == currentLine.Length - 1) { lineTokens.Add(new AiToken(AiTokenTypes.Word, "=", currentLine.Start + currentLineIndex, 1)); } else if (currentLineString[currentLineIndex + 1] == '>') { lineTokens.Add(new AiToken(AiTokenTypes.RuleArrow, "=>", currentLine.Start + currentLineIndex++, 2)); } else if (currentLineString[currentLineIndex + 1] == '=') { lineTokens.Add(new AiToken(AiTokenTypes.Word, "==", currentLine.Start + currentLineIndex++, 2)); } else // Invalid, just define this char as an invalid word and proceed scanning { lineTokens.Add(new AiToken(AiTokenTypes.Word, "=", currentLine.Start + currentLineIndex, 1)); } } else if (currentLineChar == '-' || char.IsDigit(currentLineChar)) { // Read number int numberLength = 1; while (currentLineIndex + numberLength < currentLine.Length) { // Test char if (!char.IsDigit(currentLineString[currentLineIndex + numberLength])) { break; } ++numberLength; } lineTokens.Add(new AiToken(AiTokenTypes.Number, currentLineString.Substring(currentLineIndex, numberLength), currentLine.Start + currentLineIndex, numberLength)); currentLineIndex += numberLength - 1; } else if (!char.IsWhiteSpace(currentLineChar)) { // Read word int wordLength = 1; while (currentLineIndex + wordLength < currentLine.Length) { // Test char char c = currentLineString[currentLineIndex + wordLength]; if (!char.IsLetterOrDigit(c) && c != '-' && c != '=') { break; } ++wordLength; } string word = currentLineString.Substring(currentLineIndex, wordLength); // Produce token depending on word recognition AiTokenTypes wordType = AiTokenTypes.Word; if (word == "defrule") { wordType = AiTokenTypes.Defrule; } else if (word == "defconst") { wordType = AiTokenTypes.Defconst; } else if (word == "load") { wordType = AiTokenTypes.Load; } else if (word == "load-random") { wordType = AiTokenTypes.LoadRandom; } else if (Constants.AiRuleBooleanFacts.Contains(word)) { wordType = AiTokenTypes.BooleanFactName; } else if (Constants.AiRuleFacts.ContainsKey(word)) { wordType = AiTokenTypes.FactName; } else if (Constants.AiRuleActions.ContainsKey(word)) { wordType = AiTokenTypes.ActionName; } lineTokens.Add(new AiToken(wordType, word, currentLine.Start + currentLineIndex, wordLength)); currentLineIndex += wordLength - 1; } // Next character ++currentLineIndex; } // Return the line tokens in the order determined by the parsing direction if (backwards) { for (int i = lineTokens.Count - 1; i >= 0; --i) { yield return(lineTokens[i]); } } else { foreach (AiToken lineToken in lineTokens) { yield return(lineToken); } } // Finished? if (currentLine.LineNumber == lastLineNumber) { yield break; } // Next line if (backwards) { currentLine = currentLine.Snapshot.GetLineFromLineNumber(currentLine.LineNumber - 1); } else { currentLine = currentLine.Snapshot.GetLineFromLineNumber(currentLine.LineNumber + 1); } currentLineString = currentLine.GetText(); currentLineIndex = 0; } }