public static bool GetKeywordArgumentBounds(TokenStream <RdToken> tokens, out int startTokenIndex, out int endTokenIndex)
{
startTokenIndex = -1;
endTokenIndex = -1;
TokenBraceCounter <RdToken> braceCounter = new TokenBraceCounter <RdToken>(
new RdToken(RdTokenType.OpenCurlyBrace),
new RdToken(RdTokenType.CloseCurlyBrace),
new RdToken(RdTokenType.OpenSquareBracket),
new RdToken(RdTokenType.CloseSquareBracket),
new RdTokenTypeComparer());
for (int pos = tokens.Position; pos < tokens.Length; pos++)
{
RdToken token = tokens[pos];
if (braceCounter.CountBrace(token))
{
if (startTokenIndex < 0)
{
startTokenIndex = pos;
}
if (braceCounter.Count == 0)
{
endTokenIndex = pos;
break;
}
}
}
return(startTokenIndex >= 0 && endTokenIndex >= 0 && startTokenIndex < endTokenIndex);
}
private static int SkipS3Method(RdParseContext context, ref int index)
{
RdToken token = context.Tokens[index];
Debug.Assert(token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method");
index++;
for (int i = 0; i < 2; i++)
{
if (context.Tokens[index].TokenType == RdTokenType.OpenCurlyBrace)
{
index++;
}
if (context.Tokens[index].TokenType == RdTokenType.CloseCurlyBrace)
{
index++;
}
}
// Should be past \method{...}{...}. Now skip signature
BraceCounter <char> bc = new BraceCounter <char>(new char[] { '(', ')' });
for (int i = context.Tokens[index - 1].End; i < context.TextProvider.Length; i++)
{
if (bc.CountBrace(context.TextProvider[i]))
{
if (bc.Count == 0)
{
// Calculate index of the next token after text position 'i'
index = context.Tokens.Length - 1;
for (int j = index; j < context.Tokens.Length; j++)
{
if (context.Tokens[j].Start >= i)
{
index = j;
break;
}
}
return(i + 1);
}
}
}
return(context.Tokens[index].End);
}
/// <summary>
/// Extracts R-parseable text from RD \usage{...} block.
/// RD text may contain \dots sequence which denotes ellipsis.
/// R parser does not know about it and hence we must replace \dots by ...
/// Also, signatures may contain S3 method info like
/// '\method{as.matrix}{data.frame}(x, rownames.force = NA, \dots)'
/// which we need to filter out since they are irrelevant to intellisense.
/// </summary>
private static string GetRText(RdParseContext context, int startTokenIndex, int endTokenIndex)
{
StringBuilder sb = new StringBuilder();
for (int i = startTokenIndex; i < endTokenIndex; i++)
{
int fragmentStart;
int fragmentEnd;
RdToken token = context.Tokens[i];
if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method")
{
fragmentStart = SkipS3Method(context, ref i);
fragmentEnd = context.Tokens[i].Start;
}
else
{
if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\dots")
{
sb.Append("...");
}
fragmentStart = context.Tokens[i].End;
fragmentEnd = context.Tokens[i + 1].Start;
}
Debug.Assert(fragmentStart <= fragmentEnd);
if (fragmentStart <= fragmentEnd)
{
ITextRange range = TextRange.FromBounds(fragmentStart, fragmentEnd);
string fragment = context.TextProvider.GetText(range);
sb.Append(fragment);
}
else
{
break; // Something went wrong;
}
}
return(sb.ToString().Trim());
}
private static IReadOnlyList <IFunctionInfo> ParseFunctions(RdParseContext context)
{
IReadOnlyList <ISignatureInfo> signatureInfos = null;
IReadOnlyDictionary <string, string> argumentDescriptions = null;
string functionDescription = null; // Description is normally one for all similar functions
bool isInternal = false;
string returnValue = null;
while (!context.Tokens.IsEndOfStream() && argumentDescriptions == null)
{
RdToken token = context.Tokens.CurrentToken;
if (context.IsAtKeywordWithParameters())
{
if (string.IsNullOrEmpty(functionDescription) && context.IsAtKeyword(@"\description"))
{
functionDescription = RdText.GetText(context);
}
else if (context.IsAtKeyword(@"\keyword"))
{
string keyword = RdText.GetText(context);
if (!string.IsNullOrEmpty(keyword) && keyword.Contains("internal"))
{
isInternal = true;
}
}
else if (string.IsNullOrEmpty(returnValue) && context.IsAtKeyword(@"\value"))
{
returnValue = RdText.GetText(context);
}
else if (argumentDescriptions == null && context.IsAtKeyword(@"\arguments"))
{
// Extract arguments and their descriptions
argumentDescriptions = RdArgumentDescription.ExtractArgumentDecriptions(context);
}
else if (signatureInfos == null && context.IsAtKeyword(@"\usage"))
{
// Extract signatures with function names
signatureInfos = RdFunctionSignature.ExtractSignatures(context);
}
else
{
context.Tokens.Advance(2);
}
}
else
{
context.Tokens.MoveToNextToken();
}
}
// Merge descriptions into signatures. Add all arguments
// listed in the \arguments{} section since function signature
// does not always list all possible arguments.
if (argumentDescriptions != null && signatureInfos != null)
{
foreach (ISignatureInfo sigInfo in signatureInfos)
{
// Add missing arguments from the \arguments{} section
foreach (string name in argumentDescriptions.Keys)
{
// TODO: do we need HashSet here instead? Generally arguments
// list is relatively short, about 10 items on average.
if (sigInfo.Arguments.FirstOrDefault(x => x.Name.Equals(name)) == null)
{
sigInfo.Arguments.Add(new ArgumentInfo(name));
}
}
// Relocate ..., if any, to the end
var ellipsisArgument = sigInfo.Arguments.FirstOrDefault(x => x.IsEllipsis);
if (ellipsisArgument != null)
{
int index = sigInfo.Arguments.IndexOf(ellipsisArgument);
sigInfo.Arguments.RemoveAt(index);
sigInfo.Arguments.Add(ellipsisArgument);
}
// Add description if it is not there yet
foreach (var arg in sigInfo.Arguments.Where(x => string.IsNullOrEmpty(x.Description)))
{
string description;
if (argumentDescriptions.TryGetValue(arg.Name, out description))
{
((NamedItemInfo)arg).Description = description ?? string.Empty;
}
}
}
}
// Merge signatures into function infos
Dictionary <string, FunctionInfo> functionInfos = new Dictionary <string, FunctionInfo>();
if (signatureInfos != null)
{
Dictionary <string, List <ISignatureInfo> > functionSignatures = new Dictionary <string, List <ISignatureInfo> >();
foreach (ISignatureInfo sigInfo in signatureInfos)
{
FunctionInfo functionInfo;
List <ISignatureInfo> sigList;
if (!functionInfos.TryGetValue(sigInfo.FunctionName, out functionInfo))
{
// Create function info
functionInfo = new FunctionInfo(sigInfo.FunctionName, functionDescription);
functionInfos[sigInfo.FunctionName] = functionInfo;
functionInfo.IsInternal = isInternal;
functionInfo.ReturnValue = returnValue;
// Create list of signatures for this function
sigList = new List <ISignatureInfo>();
functionSignatures[sigInfo.FunctionName] = sigList;
functionInfo.Signatures = sigList;
}
else
{
sigList = functionSignatures[sigInfo.FunctionName];
}
sigList.Add(sigInfo);
}
}
return(functionInfos.Values.ToList());
}
public static TokenAssertions <RdTokenType> Should(this RdToken token)
{
return(new TokenAssertions <RdTokenType>(token));
}
/// <summary>
/// Extracts argument names and descriptions from
/// the RD '\arguments{...} construct
/// </summary>
public static IReadOnlyDictionary <string, string> ExtractArgumentDecriptions(RdParseContext context)
{
// \arguments{
// \item{formula}{
// A linear model formula specifying the log - linear model.
// See \code{\link{ loglm} } for its interpretation.
// }
// \item{data}{
// Numeric array or data frame.In the first case it specifies the
// array of frequencies; in then second it provides the data frame
// from which the variables occurring in the formula are
// preferentially obtained in the usual way.
// }
// \item{start, param, eps, iter, print}{
// Arguments passed to \code{\link{ loglin} }.
// }
// \item{\dots}{
// arguments passed to the default method.
// }
// }
Dictionary <string, string> argumentDescriptions = new Dictionary <string, string>();
TokenStream <RdToken> tokens = context.Tokens;
// '\arguments{' is expected
Debug.Assert(tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace)
{
// Move past '\arguments'
tokens.MoveToNextToken();
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex))
{
// Now that we know bounds of \arguments{...} go through
// inner '\item' elements and fetch description and all
// argument names the description applies to.
//
// Example:
//
// \item{start, param, eps, iter, print}{Arguments
// passed to \code{\link{ loglin} }.}
//
while (!tokens.IsEndOfStream() && tokens.Position < endTokenIndex)
{
RdToken token = tokens.CurrentToken;
if (context.IsAtKeyword(@"\item"))
{
IEnumerable <IArgumentInfo> args = ParseArgumentItem(context);
if (args == null)
{
break;
}
foreach (var a in args)
{
argumentDescriptions[a.Name] = a.Description;
}
}
else
{
tokens.MoveToNextToken();
}
}
}
tokens.Position = endTokenIndex;
}
return(argumentDescriptions);
}
private static IReadOnlyList <IFunctionInfo> ParseFunctions(RdParseContext context)
{
IReadOnlyList <ISignatureInfo> signatureInfos = null;
IReadOnlyDictionary <string, string> argumentDescriptions = null;
var aliases = new List <string>();
string functionDescription = null; // Description is normally one for all similar functions
bool isInternal = false;
string returnValue = null;
string primaryName = null;
while (!context.Tokens.IsEndOfStream() &&
(functionDescription == null || argumentDescriptions == null ||
signatureInfos == null || returnValue == null))
{
RdToken token = context.Tokens.CurrentToken;
if (context.IsAtKeywordWithParameters())
{
if (string.IsNullOrEmpty(functionDescription) && context.IsAtKeyword(@"\description"))
{
functionDescription = RdText.GetText(context);
}
else if (context.IsAtKeyword(@"\keyword"))
{
string keyword = RdText.GetText(context);
if (!string.IsNullOrEmpty(keyword) && keyword.Contains("internal"))
{
isInternal = true;
}
}
else if (string.IsNullOrEmpty(returnValue) && context.IsAtKeyword(@"\value"))
{
returnValue = RdText.GetText(context);
}
else if (argumentDescriptions == null && context.IsAtKeyword(@"\arguments"))
{
// Extract arguments and their descriptions
argumentDescriptions = RdArgumentDescription.ExtractArgumentDecriptions(context);
}
else if (signatureInfos == null && context.IsAtKeyword(@"\usage"))
{
// Extract signatures with function names
signatureInfos = RdFunctionSignature.ExtractSignatures(context);
}
else if (context.IsAtKeyword(@"\alias"))
{
var alias = RdText.GetText(context);
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias);
}
}
else if (primaryName == null && context.IsAtKeyword(@"\name"))
{
primaryName = RdText.GetText(context);
}
else
{
context.Tokens.Advance(2);
}
}
else
{
context.Tokens.MoveToNextToken();
}
}
// Merge descriptions into signatures
if (argumentDescriptions != null && signatureInfos != null)
{
foreach (ISignatureInfo sigInfo in signatureInfos)
{
// Add missing arguments from the \arguments{} section
foreach (var arg in sigInfo.Arguments)
{
string description;
if (argumentDescriptions.TryGetValue(arg.Name, out description))
{
((NamedItemInfo)arg).Description = description ?? string.Empty;
}
}
}
}
// Merge signatures into function infos
var functionInfos = new Dictionary <string, FunctionInfo>();
if (signatureInfos != null)
{
var functionSignatures = new Dictionary <string, List <ISignatureInfo> >();
foreach (ISignatureInfo sigInfo in signatureInfos)
{
FunctionInfo functionInfo;
List <ISignatureInfo> sigList;
if (!functionInfos.TryGetValue(sigInfo.FunctionName, out functionInfo))
{
// Create function info
functionInfo = CreateFunctionInfo(sigInfo.FunctionName, functionDescription, returnValue, isInternal);
functionInfos[sigInfo.FunctionName] = functionInfo;
// Create list of signatures for this function
sigList = new List <ISignatureInfo>();
functionSignatures[sigInfo.FunctionName] = sigList;
functionInfo.Signatures = sigList;
}
else
{
sigList = functionSignatures[sigInfo.FunctionName];
}
sigList.Add(sigInfo);
}
}
// Propage to aliases
if (!string.IsNullOrWhiteSpace(primaryName))
{
FunctionInfo functionInfo;
if (functionInfos.TryGetValue(primaryName, out functionInfo))
{
foreach (var alias in aliases)
{
if (!functionInfos.ContainsKey(alias))
{
functionInfos[alias] = new FunctionInfo(alias, functionInfo);
}
}
}
}
return(functionInfos.Values.ToList());
}
