public static bool GetKeywordArgumentBounds(TokenStream <RdToken> tokens, out int startTokenIndex, out int endTokenIndex) { startTokenIndex = -1; endTokenIndex = -1; TokenBraceCounter <RdToken> braceCounter = new TokenBraceCounter <RdToken>( new RdToken(RdTokenType.OpenCurlyBrace), new RdToken(RdTokenType.CloseCurlyBrace), new RdToken(RdTokenType.OpenSquareBracket), new RdToken(RdTokenType.CloseSquareBracket), new RdTokenTypeComparer()); for (int pos = tokens.Position; pos < tokens.Length; pos++) { RdToken token = tokens[pos]; if (braceCounter.CountBrace(token)) { if (startTokenIndex < 0) { startTokenIndex = pos; } if (braceCounter.Count == 0) { endTokenIndex = pos; break; } } } return(startTokenIndex >= 0 && endTokenIndex >= 0 && startTokenIndex < endTokenIndex); }
private static int SkipS3Method(RdParseContext context, ref int index) { RdToken token = context.Tokens[index]; Debug.Assert(token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method"); index++; for (int i = 0; i < 2; i++) { if (context.Tokens[index].TokenType == RdTokenType.OpenCurlyBrace) { index++; } if (context.Tokens[index].TokenType == RdTokenType.CloseCurlyBrace) { index++; } } // Should be past \method{...}{...}. Now skip signature BraceCounter <char> bc = new BraceCounter <char>(new char[] { '(', ')' }); for (int i = context.Tokens[index - 1].End; i < context.TextProvider.Length; i++) { if (bc.CountBrace(context.TextProvider[i])) { if (bc.Count == 0) { // Calculate index of the next token after text position 'i' index = context.Tokens.Length - 1; for (int j = index; j < context.Tokens.Length; j++) { if (context.Tokens[j].Start >= i) { index = j; break; } } return(i + 1); } } } return(context.Tokens[index].End); }
/// <summary> /// Extracts R-parseable text from RD \usage{...} block. /// RD text may contain \dots sequence which denotes ellipsis. /// R parser does not know about it and hence we must replace \dots by ... /// Also, signatures may contain S3 method info like /// '\method{as.matrix}{data.frame}(x, rownames.force = NA, \dots)' /// which we need to filter out since they are irrelevant to intellisense. /// </summary> private static string GetRText(RdParseContext context, int startTokenIndex, int endTokenIndex) { StringBuilder sb = new StringBuilder(); for (int i = startTokenIndex; i < endTokenIndex; i++) { int fragmentStart; int fragmentEnd; RdToken token = context.Tokens[i]; if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method") { fragmentStart = SkipS3Method(context, ref i); fragmentEnd = context.Tokens[i].Start; } else { if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\dots") { sb.Append("..."); } fragmentStart = context.Tokens[i].End; fragmentEnd = context.Tokens[i + 1].Start; } Debug.Assert(fragmentStart <= fragmentEnd); if (fragmentStart <= fragmentEnd) { ITextRange range = TextRange.FromBounds(fragmentStart, fragmentEnd); string fragment = context.TextProvider.GetText(range); sb.Append(fragment); } else { break; // Something went wrong; } } return(sb.ToString().Trim()); }
private static IReadOnlyList <IFunctionInfo> ParseFunctions(RdParseContext context) { IReadOnlyList <ISignatureInfo> signatureInfos = null; IReadOnlyDictionary <string, string> argumentDescriptions = null; string functionDescription = null; // Description is normally one for all similar functions bool isInternal = false; string returnValue = null; while (!context.Tokens.IsEndOfStream() && argumentDescriptions == null) { RdToken token = context.Tokens.CurrentToken; if (context.IsAtKeywordWithParameters()) { if (string.IsNullOrEmpty(functionDescription) && context.IsAtKeyword(@"\description")) { functionDescription = RdText.GetText(context); } else if (context.IsAtKeyword(@"\keyword")) { string keyword = RdText.GetText(context); if (!string.IsNullOrEmpty(keyword) && keyword.Contains("internal")) { isInternal = true; } } else if (string.IsNullOrEmpty(returnValue) && context.IsAtKeyword(@"\value")) { returnValue = RdText.GetText(context); } else if (argumentDescriptions == null && context.IsAtKeyword(@"\arguments")) { // Extract arguments and their descriptions argumentDescriptions = RdArgumentDescription.ExtractArgumentDecriptions(context); } else if (signatureInfos == null && context.IsAtKeyword(@"\usage")) { // Extract signatures with function names signatureInfos = RdFunctionSignature.ExtractSignatures(context); } else { context.Tokens.Advance(2); } } else { context.Tokens.MoveToNextToken(); } } // Merge descriptions into signatures. Add all arguments // listed in the \arguments{} section since function signature // does not always list all possible arguments. if (argumentDescriptions != null && signatureInfos != null) { foreach (ISignatureInfo sigInfo in signatureInfos) { // Add missing arguments from the \arguments{} section foreach (string name in argumentDescriptions.Keys) { // TODO: do we need HashSet here instead? Generally arguments // list is relatively short, about 10 items on average. if (sigInfo.Arguments.FirstOrDefault(x => x.Name.Equals(name)) == null) { sigInfo.Arguments.Add(new ArgumentInfo(name)); } } // Relocate ..., if any, to the end var ellipsisArgument = sigInfo.Arguments.FirstOrDefault(x => x.IsEllipsis); if (ellipsisArgument != null) { int index = sigInfo.Arguments.IndexOf(ellipsisArgument); sigInfo.Arguments.RemoveAt(index); sigInfo.Arguments.Add(ellipsisArgument); } // Add description if it is not there yet foreach (var arg in sigInfo.Arguments.Where(x => string.IsNullOrEmpty(x.Description))) { string description; if (argumentDescriptions.TryGetValue(arg.Name, out description)) { ((NamedItemInfo)arg).Description = description ?? string.Empty; } } } } // Merge signatures into function infos Dictionary <string, FunctionInfo> functionInfos = new Dictionary <string, FunctionInfo>(); if (signatureInfos != null) { Dictionary <string, List <ISignatureInfo> > functionSignatures = new Dictionary <string, List <ISignatureInfo> >(); foreach (ISignatureInfo sigInfo in signatureInfos) { FunctionInfo functionInfo; List <ISignatureInfo> sigList; if (!functionInfos.TryGetValue(sigInfo.FunctionName, out functionInfo)) { // Create function info functionInfo = new FunctionInfo(sigInfo.FunctionName, functionDescription); functionInfos[sigInfo.FunctionName] = functionInfo; functionInfo.IsInternal = isInternal; functionInfo.ReturnValue = returnValue; // Create list of signatures for this function sigList = new List <ISignatureInfo>(); functionSignatures[sigInfo.FunctionName] = sigList; functionInfo.Signatures = sigList; } else { sigList = functionSignatures[sigInfo.FunctionName]; } sigList.Add(sigInfo); } } return(functionInfos.Values.ToList()); }
public static TokenAssertions <RdTokenType> Should(this RdToken token) { return(new TokenAssertions <RdTokenType>(token)); }
/// <summary> /// Extracts argument names and descriptions from /// the RD '\arguments{...} construct /// </summary> public static IReadOnlyDictionary <string, string> ExtractArgumentDecriptions(RdParseContext context) { // \arguments{ // \item{formula}{ // A linear model formula specifying the log - linear model. // See \code{\link{ loglm} } for its interpretation. // } // \item{data}{ // Numeric array or data frame.In the first case it specifies the // array of frequencies; in then second it provides the data frame // from which the variables occurring in the formula are // preferentially obtained in the usual way. // } // \item{start, param, eps, iter, print}{ // Arguments passed to \code{\link{ loglin} }. // } // \item{\dots}{ // arguments passed to the default method. // } // } Dictionary <string, string> argumentDescriptions = new Dictionary <string, string>(); TokenStream <RdToken> tokens = context.Tokens; // '\arguments{' is expected Debug.Assert(tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace); if (tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace) { // Move past '\arguments' tokens.MoveToNextToken(); int startTokenIndex, endTokenIndex; if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex)) { // Now that we know bounds of \arguments{...} go through // inner '\item' elements and fetch description and all // argument names the description applies to. // // Example: // // \item{start, param, eps, iter, print}{Arguments // passed to \code{\link{ loglin} }.} // while (!tokens.IsEndOfStream() && tokens.Position < endTokenIndex) { RdToken token = tokens.CurrentToken; if (context.IsAtKeyword(@"\item")) { IEnumerable <IArgumentInfo> args = ParseArgumentItem(context); if (args == null) { break; } foreach (var a in args) { argumentDescriptions[a.Name] = a.Description; } } else { tokens.MoveToNextToken(); } } } tokens.Position = endTokenIndex; } return(argumentDescriptions); }
private static IReadOnlyList <IFunctionInfo> ParseFunctions(RdParseContext context) { IReadOnlyList <ISignatureInfo> signatureInfos = null; IReadOnlyDictionary <string, string> argumentDescriptions = null; var aliases = new List <string>(); string functionDescription = null; // Description is normally one for all similar functions bool isInternal = false; string returnValue = null; string primaryName = null; while (!context.Tokens.IsEndOfStream() && (functionDescription == null || argumentDescriptions == null || signatureInfos == null || returnValue == null)) { RdToken token = context.Tokens.CurrentToken; if (context.IsAtKeywordWithParameters()) { if (string.IsNullOrEmpty(functionDescription) && context.IsAtKeyword(@"\description")) { functionDescription = RdText.GetText(context); } else if (context.IsAtKeyword(@"\keyword")) { string keyword = RdText.GetText(context); if (!string.IsNullOrEmpty(keyword) && keyword.Contains("internal")) { isInternal = true; } } else if (string.IsNullOrEmpty(returnValue) && context.IsAtKeyword(@"\value")) { returnValue = RdText.GetText(context); } else if (argumentDescriptions == null && context.IsAtKeyword(@"\arguments")) { // Extract arguments and their descriptions argumentDescriptions = RdArgumentDescription.ExtractArgumentDecriptions(context); } else if (signatureInfos == null && context.IsAtKeyword(@"\usage")) { // Extract signatures with function names signatureInfos = RdFunctionSignature.ExtractSignatures(context); } else if (context.IsAtKeyword(@"\alias")) { var alias = RdText.GetText(context); if (!string.IsNullOrWhiteSpace(alias)) { aliases.Add(alias); } } else if (primaryName == null && context.IsAtKeyword(@"\name")) { primaryName = RdText.GetText(context); } else { context.Tokens.Advance(2); } } else { context.Tokens.MoveToNextToken(); } } // Merge descriptions into signatures if (argumentDescriptions != null && signatureInfos != null) { foreach (ISignatureInfo sigInfo in signatureInfos) { // Add missing arguments from the \arguments{} section foreach (var arg in sigInfo.Arguments) { string description; if (argumentDescriptions.TryGetValue(arg.Name, out description)) { ((NamedItemInfo)arg).Description = description ?? string.Empty; } } } } // Merge signatures into function infos var functionInfos = new Dictionary <string, FunctionInfo>(); if (signatureInfos != null) { var functionSignatures = new Dictionary <string, List <ISignatureInfo> >(); foreach (ISignatureInfo sigInfo in signatureInfos) { FunctionInfo functionInfo; List <ISignatureInfo> sigList; if (!functionInfos.TryGetValue(sigInfo.FunctionName, out functionInfo)) { // Create function info functionInfo = CreateFunctionInfo(sigInfo.FunctionName, functionDescription, returnValue, isInternal); functionInfos[sigInfo.FunctionName] = functionInfo; // Create list of signatures for this function sigList = new List <ISignatureInfo>(); functionSignatures[sigInfo.FunctionName] = sigList; functionInfo.Signatures = sigList; } else { sigList = functionSignatures[sigInfo.FunctionName]; } sigList.Add(sigInfo); } } // Propage to aliases if (!string.IsNullOrWhiteSpace(primaryName)) { FunctionInfo functionInfo; if (functionInfos.TryGetValue(primaryName, out functionInfo)) { foreach (var alias in aliases) { if (!functionInfos.ContainsKey(alias)) { functionInfos[alias] = new FunctionInfo(alias, functionInfo); } } } } return(functionInfos.Values.ToList()); }