/// <summary> /// Gets a list of tokens from the IVsTextLines. /// </summary> /// <param name="lines">The IVsTextLines of the lines of source to parse.</param> /// <param name="service">The language service.</param> /// <returns>A list of tokens.</returns> public static List <FactTokenInfo> GetTokens(IVsTextLines lines, FactEditorLanguageService service) { Source source = service.GetOrCreateSource(lines); string lineText = source.GetLine(0); // Get the scanner from the language service. IScanner scanner = service.GetScanner(lines); scanner.SetSource(lineText, 0); // Now use the scanner to parse the first line and build the list of the tokens. List <FactTokenInfo> tokens = new List <FactTokenInfo>(); FactTokenInfo lastToken = null; FactTokenInfo currentToken = new FactTokenInfo(); int state = 0; while (scanner.ScanTokenAndProvideInfoAboutIt(currentToken, ref state)) { if ((null != lastToken) && (currentToken.StartIndex > lastToken.EndIndex + 1)) { tokens.Clear(); } tokens.Add(currentToken); lastToken = currentToken; currentToken = new FactTokenInfo(); } return(tokens); }
/// <summary> /// Gets a list of tokens from the IVsTextLines. /// </summary> /// <param name="lines">The IVsTextLines of the lines of source to parse.</param> /// <param name="service">The language service.</param> /// <returns>A list of tokens.</returns> public static List<FactTokenInfo> GetTokens(IVsTextLines lines, FactEditorLanguageService service) { Source source = service.GetOrCreateSource(lines); string lineText = source.GetLine(0); // Get the scanner from the language service. IScanner scanner = service.GetScanner(lines); scanner.SetSource(lineText, 0); // Now use the scanner to parse the first line and build the list of the tokens. List<FactTokenInfo> tokens = new List<FactTokenInfo>(); FactTokenInfo lastToken = null; FactTokenInfo currentToken = new FactTokenInfo(); int state = 0; while (scanner.ScanTokenAndProvideInfoAboutIt(currentToken, ref state)) { if ((null != lastToken) && (currentToken.StartIndex > lastToken.EndIndex + 1)) { tokens.Clear(); } tokens.Add(currentToken); lastToken = currentToken; currentToken = new FactTokenInfo(); } return tokens; }