public static IReadOnlyList<ISignatureInfo> ExtractSignatures(RdParseContext context) {
// \usage{
// loglm1(formula, data, \dots)
// \method{loglm1}{xtabs}(formula, data, \dots)
// \method{loglm1}{data.frame}(formula, data, \dots)
// \method{loglm1}{default}(formula, data, start = rep(1, length(data)), fitted = FALSE,
// keep.frequencies = fitted, param = TRUE, eps = 1 / 10,
// iter = 40, print = FALSE, \dots)
// }
//
// Signatures can be for multiple related functions
// }\usage{
// lockEnvironment(env, bindings = FALSE)
// environmentIsLocked(env)
// lockBinding(sym, env)
// unlockBinding(sym, env)
// bindingIsLocked(sym, env)
TokenStream<RdToken> tokens = context.Tokens;
List<ISignatureInfo> signatures = new List<ISignatureInfo>();
// Must be at '\usage{'
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex)) {
// Get inner content of the \usage{...} block cleaned up for R parsing
string usage = GetRText(context, startTokenIndex, endTokenIndex);
IReadOnlyList<ISignatureInfo> sigs = ParseSignatures(usage);
if (sigs != null) {
signatures.AddRange(sigs);
}
tokens.Position = endTokenIndex;
}
return signatures;
}
/// <summary>
/// Extracts R-parseable text from RD \usage{...} block.
/// RD text may contain \dots sequence which denotes ellipsis.
/// R parser does not know about it and hence we must replace \dots by ...
/// Also, signatures may contain S3 method info like
/// '\method{as.matrix}{data.frame}(x, rownames.force = NA, \dots)'
/// which we need to filter out since they are irrelevant to intellisense.
/// </summary>
private static string GetRText(RdParseContext context, int startTokenIndex, int endTokenIndex) {
StringBuilder sb = new StringBuilder();
for (int i = startTokenIndex; i < endTokenIndex; i++) {
int fragmentStart;
int fragmentEnd;
RdToken token = context.Tokens[i];
if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method") {
fragmentStart = SkipS3Method(context, ref i);
fragmentEnd = context.Tokens[i].Start;
} else {
if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\dots") {
sb.Append("...");
}
fragmentStart = context.Tokens[i].End;
fragmentEnd = context.Tokens[i + 1].Start;
}
Debug.Assert(fragmentStart <= fragmentEnd);
if (fragmentStart <= fragmentEnd) {
ITextRange range = TextRange.FromBounds(fragmentStart, fragmentEnd);
string fragment = context.TextProvider.GetText(range);
sb.Append(fragment);
}
else {
break; // Something went wrong;
}
}
return sb.ToString().Trim();
}
public static string FromTokens(RdParseContext context, int startTokenIndex, int endTokenIndex)
{
Debug.Assert(startTokenIndex >= 0 && startTokenIndex < endTokenIndex);
// Clean descripton so it only consists of plain text
var sb = new StringBuilder();
for (int i = startTokenIndex; i < endTokenIndex; i++)
{
TextRange range = TextRange.FromBounds(context.Tokens[i].End, context.Tokens[i + 1].Start);
string s = context.TextProvider.GetText(range);
s = CleanRawRdText(s);
if (!string.IsNullOrWhiteSpace(s) &&
(sb.Length > 0 && !char.IsWhiteSpace(sb[sb.Length - 1]) &&
char.IsLetterOrDigit(s[0])))
{
sb.Append(' ');
}
sb.Append(s);
}
return(sb.ToString());
}
/// <summary>
/// Given RD data and function name parses the data and creates structured
/// information about the function. Method returns multiple functions since
/// RD data often provides information on several functions so in order
/// to avoid processing same data multiple times parser extracts information
/// on all related functions.
/// </summary>
public static IReadOnlyList<IFunctionInfo> GetFunctionInfos(string rdHelpData) {
var tokenizer = new RdTokenizer(tokenizeRContent: false);
ITextProvider textProvider = new TextStream(rdHelpData);
IReadOnlyTextRangeCollection<RdToken> tokens = tokenizer.Tokenize(textProvider, 0, textProvider.Length);
RdParseContext context = new RdParseContext(tokens, textProvider);
return ParseFunctions(context);
}
/// <summary>
/// Given RD data and function name parses the data and creates structured
/// information about the function. Method returns multiple functions since
/// RD data often provides information on several functions so in order
/// to avoid processing same data multiple times parser extracts information
/// on all related functions.
/// </summary>
public static IReadOnlyList <IFunctionInfo> GetFunctionInfos(string rdHelpData)
{
var tokenizer = new RdTokenizer(tokenizeRContent: false);
ITextProvider textProvider = new TextStream(rdHelpData);
IReadOnlyTextRangeCollection <RdToken> tokens = tokenizer.Tokenize(textProvider, 0, textProvider.Length);
RdParseContext context = new RdParseContext(tokens, textProvider);
return(ParseFunctions(context));
}
public static string GetText(RdParseContext context) {
string text = string.Empty;
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(context.Tokens, out startTokenIndex, out endTokenIndex)) {
text = RdText.FromTokens(context, startTokenIndex, endTokenIndex);
context.Tokens.Position = endTokenIndex;
}
return text.Trim();
}
public static string GetText(RdParseContext context)
{
string text = string.Empty;
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(context.Tokens, out startTokenIndex, out endTokenIndex))
{
text = RdText.FromTokens(context, startTokenIndex, endTokenIndex);
context.Tokens.Position = endTokenIndex;
}
return(text);
}
public static string FromTokens(RdParseContext context, int startTokenIndex, int endTokenIndex) {
Debug.Assert(startTokenIndex >= 0 && startTokenIndex < endTokenIndex);
// Clean descripton so it only consists of plain text
var sb = new StringBuilder();
for (int i = startTokenIndex; i < endTokenIndex; i++) {
TextRange range = TextRange.FromBounds(context.Tokens[i].End, context.Tokens[i + 1].Start);
string s = context.TextProvider.GetText(range);
s = CleanRawRdText(s);
sb.Append(s);
}
return sb.ToString();
}
private static IEnumerable <IArgumentInfo> ParseArgumentItem(RdParseContext context)
{
List <IArgumentInfo> arguments = null;
TokenStream <RdToken> tokens = context.Tokens;
tokens.Advance(1);
// Past '\item'. Inside { } we can find any number of '\dots' which are keywords.
Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace)
{
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex))
{
TextRange range = TextRange.FromBounds(tokens[startTokenIndex].End, tokens[endTokenIndex].Start);
string argumentsText = context.TextProvider.GetText(range);
string[] argumentNames = argumentsText.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
arguments = new List <IArgumentInfo>();
// Move past \item{}
tokens.Position = endTokenIndex + 1;
Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace)
{
string description = RdText.GetText(context);
foreach (string n in argumentNames)
{
string name = n.Trim();
if (name == @"\dots")
{
name = "...";
}
ArgumentInfo info = new ArgumentInfo(name, description.Trim());
arguments.Add(info);
}
}
}
}
return(arguments);
}
public static string FromTokens(RdParseContext context, int startTokenIndex, int endTokenIndex)
{
Debug.Assert(startTokenIndex >= 0 && startTokenIndex < endTokenIndex);
// Clean descripton so it only consists of plain text
var sb = new StringBuilder();
for (int i = startTokenIndex; i < endTokenIndex; i++)
{
TextRange range = TextRange.FromBounds(context.Tokens[i].End, context.Tokens[i + 1].Start);
string s = context.TextProvider.GetText(range);
s = CleanRawRdText(s);
sb.Append(s);
}
return(sb.ToString());
}
private static int SkipS3Method(RdParseContext context, ref int index)
{
RdToken token = context.Tokens[index];
Debug.Assert(token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method");
index++;
for (int i = 0; i < 2; i++)
{
if (context.Tokens[index].TokenType == RdTokenType.OpenCurlyBrace)
{
index++;
}
if (context.Tokens[index].TokenType == RdTokenType.CloseCurlyBrace)
{
index++;
}
}
// Should be past \method{...}{...}. Now skip signature
BraceCounter <char> bc = new BraceCounter <char>(new char[] { '(', ')' });
for (int i = context.Tokens[index - 1].End; i < context.TextProvider.Length; i++)
{
if (bc.CountBrace(context.TextProvider[i]))
{
if (bc.Count == 0)
{
// Calculate index of the next token after text position 'i'
index = context.Tokens.Length - 1;
for (int j = index; j < context.Tokens.Length; j++)
{
if (context.Tokens[j].Start >= i)
{
index = j;
break;
}
}
return(i + 1);
}
}
}
return(context.Tokens[index].End);
}
private static IEnumerable<IArgumentInfo> ParseArgumentItem(RdParseContext context) {
List<IArgumentInfo> arguments = null;
TokenStream<RdToken> tokens = context.Tokens;
tokens.Advance(1);
// Past '\item'. Inside { } we can find any number of '\dots' which are keywords.
Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace) {
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex)) {
TextRange range = TextRange.FromBounds(tokens[startTokenIndex].End, tokens[endTokenIndex].Start);
string argumentsText = context.TextProvider.GetText(range);
string[] argumentNames = argumentsText.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
arguments = new List<IArgumentInfo>();
// Move past \item{}
tokens.Position = endTokenIndex + 1;
Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace) {
string description = RdText.GetText(context);
foreach (string n in argumentNames) {
string name = n.Trim();
if (name == @"\dots") {
name = "...";
}
ArgumentInfo info = new ArgumentInfo(name, description.Trim());
arguments.Add(info);
}
}
}
}
return arguments;
}
public static string FromTokens(RdParseContext context, int startTokenIndex, int endTokenIndex) {
Debug.Assert(startTokenIndex >= 0 && startTokenIndex < endTokenIndex);
// Clean descripton so it only consists of plain text
var sb = new StringBuilder();
for (int i = startTokenIndex; i < endTokenIndex; i++) {
TextRange range = TextRange.FromBounds(context.Tokens[i].End, context.Tokens[i + 1].Start);
string s = context.TextProvider.GetText(range);
s = CleanRawRdText(s);
if (!string.IsNullOrWhiteSpace(s) &&
(sb.Length > 0 && !char.IsWhiteSpace(sb[sb.Length - 1]) &&
char.IsLetterOrDigit(s[0]))) {
sb.Append(' ');
}
sb.Append(s);
}
return sb.ToString();
}
/// <summary>
/// Extracts R-parseable text from RD \usage{...} block.
/// RD text may contain \dots sequence which denotes ellipsis.
/// R parser does not know about it and hence we must replace \dots by ...
/// Also, signatures may contain S3 method info like
/// '\method{as.matrix}{data.frame}(x, rownames.force = NA, \dots)'
/// which we need to filter out since they are irrelevant to intellisense.
/// </summary>
private static string GetRText(RdParseContext context, int startTokenIndex, int endTokenIndex)
{
StringBuilder sb = new StringBuilder();
for (int i = startTokenIndex; i < endTokenIndex; i++)
{
int fragmentStart;
int fragmentEnd;
RdToken token = context.Tokens[i];
if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method")
{
fragmentStart = SkipS3Method(context, ref i);
fragmentEnd = context.Tokens[i].Start;
}
else
{
if (token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\dots")
{
sb.Append("...");
}
fragmentStart = context.Tokens[i].End;
fragmentEnd = context.Tokens[i + 1].Start;
}
Debug.Assert(fragmentStart <= fragmentEnd);
if (fragmentStart <= fragmentEnd)
{
ITextRange range = TextRange.FromBounds(fragmentStart, fragmentEnd);
string fragment = context.TextProvider.GetText(range);
sb.Append(fragment);
}
else
{
break; // Something went wrong;
}
}
return(sb.ToString().Trim());
}
public static IReadOnlyList <ISignatureInfo> ExtractSignatures(RdParseContext context)
{
// \usage{
// loglm1(formula, data, \dots)
// \method{loglm1}{xtabs}(formula, data, \dots)
// \method{loglm1}{data.frame}(formula, data, \dots)
// \method{loglm1}{default}(formula, data, start = rep(1, length(data)), fitted = FALSE,
// keep.frequencies = fitted, param = TRUE, eps = 1 / 10,
// iter = 40, print = FALSE, \dots)
// }
//
// Signatures can be for multiple related functions
// }\usage{
// lockEnvironment(env, bindings = FALSE)
// environmentIsLocked(env)
// lockBinding(sym, env)
// unlockBinding(sym, env)
// bindingIsLocked(sym, env)
TokenStream <RdToken> tokens = context.Tokens;
List <ISignatureInfo> signatures = new List <ISignatureInfo>();
// Must be at '\usage{'
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex))
{
// Get inner content of the \usage{...} block cleaned up for R parsing
string usage = GetRText(context, startTokenIndex, endTokenIndex);
IReadOnlyList <ISignatureInfo> sigs = ParseSignatures(usage);
if (sigs != null)
{
signatures.AddRange(sigs);
}
tokens.Position = endTokenIndex;
}
return(signatures);
}
private static IReadOnlyList <IFunctionInfo> ParseFunctions(RdParseContext context)
{
IReadOnlyList <ISignatureInfo> signatureInfos = null;
IReadOnlyDictionary <string, string> argumentDescriptions = null;
string functionDescription = null; // Description is normally one for all similar functions
bool isInternal = false;
string returnValue = null;
while (!context.Tokens.IsEndOfStream() && argumentDescriptions == null)
{
RdToken token = context.Tokens.CurrentToken;
if (context.IsAtKeywordWithParameters())
{
if (string.IsNullOrEmpty(functionDescription) && context.IsAtKeyword(@"\description"))
{
functionDescription = RdText.GetText(context);
}
else if (context.IsAtKeyword(@"\keyword"))
{
string keyword = RdText.GetText(context);
if (!string.IsNullOrEmpty(keyword) && keyword.Contains("internal"))
{
isInternal = true;
}
}
else if (string.IsNullOrEmpty(returnValue) && context.IsAtKeyword(@"\value"))
{
returnValue = RdText.GetText(context);
}
else if (argumentDescriptions == null && context.IsAtKeyword(@"\arguments"))
{
// Extract arguments and their descriptions
argumentDescriptions = RdArgumentDescription.ExtractArgumentDecriptions(context);
}
else if (signatureInfos == null && context.IsAtKeyword(@"\usage"))
{
// Extract signatures with function names
signatureInfos = RdFunctionSignature.ExtractSignatures(context);
}
else
{
context.Tokens.Advance(2);
}
}
else
{
context.Tokens.MoveToNextToken();
}
}
// Merge descriptions into signatures. Add all arguments
// listed in the \arguments{} section since function signature
// does not always list all possible arguments.
if (argumentDescriptions != null && signatureInfos != null)
{
foreach (ISignatureInfo sigInfo in signatureInfos)
{
// Add missing arguments from the \arguments{} section
foreach (string name in argumentDescriptions.Keys)
{
// TODO: do we need HashSet here instead? Generally arguments
// list is relatively short, about 10 items on average.
if (sigInfo.Arguments.FirstOrDefault(x => x.Name.Equals(name)) == null)
{
sigInfo.Arguments.Add(new ArgumentInfo(name));
}
}
// Relocate ..., if any, to the end
var ellipsisArgument = sigInfo.Arguments.FirstOrDefault(x => x.IsEllipsis);
if (ellipsisArgument != null)
{
int index = sigInfo.Arguments.IndexOf(ellipsisArgument);
sigInfo.Arguments.RemoveAt(index);
sigInfo.Arguments.Add(ellipsisArgument);
}
// Add description if it is not there yet
foreach (var arg in sigInfo.Arguments.Where(x => string.IsNullOrEmpty(x.Description)))
{
string description;
if (argumentDescriptions.TryGetValue(arg.Name, out description))
{
((NamedItemInfo)arg).Description = description ?? string.Empty;
}
}
}
}
// Merge signatures into function infos
Dictionary <string, FunctionInfo> functionInfos = new Dictionary <string, FunctionInfo>();
if (signatureInfos != null)
{
Dictionary <string, List <ISignatureInfo> > functionSignatures = new Dictionary <string, List <ISignatureInfo> >();
foreach (ISignatureInfo sigInfo in signatureInfos)
{
FunctionInfo functionInfo;
List <ISignatureInfo> sigList;
if (!functionInfos.TryGetValue(sigInfo.FunctionName, out functionInfo))
{
// Create function info
functionInfo = new FunctionInfo(sigInfo.FunctionName, functionDescription);
functionInfos[sigInfo.FunctionName] = functionInfo;
functionInfo.IsInternal = isInternal;
functionInfo.ReturnValue = returnValue;
// Create list of signatures for this function
sigList = new List <ISignatureInfo>();
functionSignatures[sigInfo.FunctionName] = sigList;
functionInfo.Signatures = sigList;
}
else
{
sigList = functionSignatures[sigInfo.FunctionName];
}
sigList.Add(sigInfo);
}
}
return(functionInfos.Values.ToList());
}
private static IReadOnlyList<IFunctionInfo> ParseFunctions(RdParseContext context) {
IReadOnlyList<ISignatureInfo> signatureInfos = null;
IReadOnlyDictionary<string, string> argumentDescriptions = null;
string functionDescription = null; // Description is normally one for all similar functions
bool isInternal = false;
string returnValue = null;
while (!context.Tokens.IsEndOfStream() && argumentDescriptions == null) {
RdToken token = context.Tokens.CurrentToken;
if (context.IsAtKeywordWithParameters()) {
if (string.IsNullOrEmpty(functionDescription) && context.IsAtKeyword(@"\description")) {
functionDescription = RdText.GetText(context);
} else if (context.IsAtKeyword(@"\keyword")) {
string keyword = RdText.GetText(context);
if (!string.IsNullOrEmpty(keyword) && keyword.Contains("internal")) {
isInternal = true;
}
} else if (string.IsNullOrEmpty(returnValue) && context.IsAtKeyword(@"\value")) {
returnValue = RdText.GetText(context);
} else if (argumentDescriptions == null && context.IsAtKeyword(@"\arguments")) {
// Extract arguments and their descriptions
argumentDescriptions = RdArgumentDescription.ExtractArgumentDecriptions(context);
} else if (signatureInfos == null && context.IsAtKeyword(@"\usage")) {
// Extract signatures with function names
signatureInfos = RdFunctionSignature.ExtractSignatures(context);
} else {
context.Tokens.Advance(2);
}
} else {
context.Tokens.MoveToNextToken();
}
}
// Merge descriptions into signatures. Add all arguments
// listed in the \arguments{} section since function signature
// does not always list all possible arguments.
if (argumentDescriptions != null && signatureInfos != null) {
foreach (ISignatureInfo sigInfo in signatureInfos) {
// Add missing arguments from the \arguments{} section
foreach (string name in argumentDescriptions.Keys) {
// TODO: do we need HashSet here instead? Generally arguments
// list is relatively short, about 10 items on average.
if (sigInfo.Arguments.FirstOrDefault(x => x.Name.Equals(name)) == null) {
sigInfo.Arguments.Add(new ArgumentInfo(name));
}
}
// Relocate ..., if any, to the end
var ellipsisArgument = sigInfo.Arguments.FirstOrDefault(x => x.IsEllipsis);
if (ellipsisArgument != null) {
int index = sigInfo.Arguments.IndexOf(ellipsisArgument);
sigInfo.Arguments.RemoveAt(index);
sigInfo.Arguments.Add(ellipsisArgument);
}
// Add description if it is not there yet
foreach (var arg in sigInfo.Arguments.Where(x => string.IsNullOrEmpty(x.Description))) {
string description;
if (argumentDescriptions.TryGetValue(arg.Name, out description)) {
((NamedItemInfo)arg).Description = description ?? string.Empty;
}
}
}
}
// Merge signatures into function infos
Dictionary<string, FunctionInfo> functionInfos = new Dictionary<string, FunctionInfo>();
if (signatureInfos != null) {
Dictionary<string, List<ISignatureInfo>> functionSignatures = new Dictionary<string, List<ISignatureInfo>>();
foreach (ISignatureInfo sigInfo in signatureInfos) {
FunctionInfo functionInfo;
List<ISignatureInfo> sigList;
if (!functionInfos.TryGetValue(sigInfo.FunctionName, out functionInfo)) {
// Create function info
functionInfo = new FunctionInfo(sigInfo.FunctionName, functionDescription);
functionInfos[sigInfo.FunctionName] = functionInfo;
functionInfo.IsInternal = isInternal;
functionInfo.ReturnValue = returnValue;
// Create list of signatures for this function
sigList = new List<ISignatureInfo>();
functionSignatures[sigInfo.FunctionName] = sigList;
functionInfo.Signatures = sigList;
} else {
sigList = functionSignatures[sigInfo.FunctionName];
}
sigList.Add(sigInfo);
}
}
return functionInfos.Values.ToList();
}
private static int SkipS3Method(RdParseContext context, ref int index) {
RdToken token = context.Tokens[index];
Debug.Assert(token.TokenType == RdTokenType.Keyword && context.TextProvider.GetText(token) == "\\method");
index++;
for (int i = 0; i < 2; i++) {
if (context.Tokens[index].TokenType == RdTokenType.OpenCurlyBrace) {
index++;
}
if (context.Tokens[index].TokenType == RdTokenType.CloseCurlyBrace) {
index++;
}
}
// Should be past \method{...}{...}. Now skip signature
BraceCounter<char> bc = new BraceCounter<char>(new char[] { '(', ')' });
for (int i = context.Tokens[index - 1].End; i < context.TextProvider.Length; i++) {
if (bc.CountBrace(context.TextProvider[i])) {
if (bc.Count == 0) {
// Calculate index of the next token after text position 'i'
index = context.Tokens.Length - 1;
for (int j = index; j < context.Tokens.Length; j++) {
if (context.Tokens[j].Start >= i) {
index = j;
break;
}
}
return i + 1;
}
}
}
return context.Tokens[index].End;
}
/// <summary>
/// Extracts argument names and descriptions from
/// the RD '\arguments{...} construct
/// </summary>
public static IReadOnlyDictionary<string, string> ExtractArgumentDecriptions(RdParseContext context) {
// \arguments{
// \item{formula}{
// A linear model formula specifying the log - linear model.
// See \code{\link{ loglm} } for its interpretation.
// }
// \item{data}{
// Numeric array or data frame.In the first case it specifies the
// array of frequencies; in then second it provides the data frame
// from which the variables occurring in the formula are
// preferentially obtained in the usual way.
// }
// \item{start, param, eps, iter, print}{
// Arguments passed to \code{\link{ loglin} }.
// }
// \item{\dots}{
// arguments passed to the default method.
// }
// }
Dictionary<string, string> argumentDescriptions = new Dictionary<string, string>();
TokenStream<RdToken> tokens = context.Tokens;
// '\arguments{' is expected
Debug.Assert(tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace) {
// Move past '\arguments'
tokens.MoveToNextToken();
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex)) {
// Now that we know bounds of \arguments{...} go through
// inner '\item' elements and fetch description and all
// argument names the description applies to.
//
// Example:
//
// \item{start, param, eps, iter, print}{Arguments
// passed to \code{\link{ loglin} }.}
//
while (!tokens.IsEndOfStream() && tokens.Position < endTokenIndex) {
RdToken token = tokens.CurrentToken;
if (context.IsAtKeyword(@"\item")) {
IEnumerable<IArgumentInfo> args = ParseArgumentItem(context);
if (args == null)
break;
foreach (var a in args) {
argumentDescriptions[a.Name] = a.Description;
}
} else {
tokens.MoveToNextToken();
}
}
}
tokens.Position = endTokenIndex;
}
return argumentDescriptions;
}
/// <summary>
/// Extracts argument names and descriptions from
/// the RD '\arguments{...} construct
/// </summary>
public static IReadOnlyDictionary <string, string> ExtractArgumentDecriptions(RdParseContext context)
{
// \arguments{
// \item{formula}{
// A linear model formula specifying the log - linear model.
// See \code{\link{ loglm} } for its interpretation.
// }
// \item{data}{
// Numeric array or data frame.In the first case it specifies the
// array of frequencies; in then second it provides the data frame
// from which the variables occurring in the formula are
// preferentially obtained in the usual way.
// }
// \item{start, param, eps, iter, print}{
// Arguments passed to \code{\link{ loglin} }.
// }
// \item{\dots}{
// arguments passed to the default method.
// }
// }
Dictionary <string, string> argumentDescriptions = new Dictionary <string, string>();
TokenStream <RdToken> tokens = context.Tokens;
// '\arguments{' is expected
Debug.Assert(tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.NextToken.TokenType == RdTokenType.OpenCurlyBrace)
{
// Move past '\arguments'
tokens.MoveToNextToken();
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex))
{
// Now that we know bounds of \arguments{...} go through
// inner '\item' elements and fetch description and all
// argument names the description applies to.
//
// Example:
//
// \item{start, param, eps, iter, print}{Arguments
// passed to \code{\link{ loglin} }.}
//
while (!tokens.IsEndOfStream() && tokens.Position < endTokenIndex)
{
RdToken token = tokens.CurrentToken;
if (context.IsAtKeyword(@"\item"))
{
IEnumerable <IArgumentInfo> args = ParseArgumentItem(context);
if (args == null)
{
break;
}
foreach (var a in args)
{
argumentDescriptions[a.Name] = a.Description;
}
}
else
{
tokens.MoveToNextToken();
}
}
}
tokens.Position = endTokenIndex;
}
return(argumentDescriptions);
}
private static IReadOnlyList <IFunctionInfo> ParseFunctions(RdParseContext context)
{
IReadOnlyList <ISignatureInfo> signatureInfos = null;
IReadOnlyDictionary <string, string> argumentDescriptions = null;
var aliases = new List <string>();
string functionDescription = null; // Description is normally one for all similar functions
bool isInternal = false;
string returnValue = null;
string primaryName = null;
while (!context.Tokens.IsEndOfStream() &&
(functionDescription == null || argumentDescriptions == null ||
signatureInfos == null || returnValue == null))
{
RdToken token = context.Tokens.CurrentToken;
if (context.IsAtKeywordWithParameters())
{
if (string.IsNullOrEmpty(functionDescription) && context.IsAtKeyword(@"\description"))
{
functionDescription = RdText.GetText(context);
}
else if (context.IsAtKeyword(@"\keyword"))
{
string keyword = RdText.GetText(context);
if (!string.IsNullOrEmpty(keyword) && keyword.Contains("internal"))
{
isInternal = true;
}
}
else if (string.IsNullOrEmpty(returnValue) && context.IsAtKeyword(@"\value"))
{
returnValue = RdText.GetText(context);
}
else if (argumentDescriptions == null && context.IsAtKeyword(@"\arguments"))
{
// Extract arguments and their descriptions
argumentDescriptions = RdArgumentDescription.ExtractArgumentDecriptions(context);
}
else if (signatureInfos == null && context.IsAtKeyword(@"\usage"))
{
// Extract signatures with function names
signatureInfos = RdFunctionSignature.ExtractSignatures(context);
}
else if (context.IsAtKeyword(@"\alias"))
{
var alias = RdText.GetText(context);
if (!string.IsNullOrWhiteSpace(alias))
{
aliases.Add(alias);
}
}
else if (primaryName == null && context.IsAtKeyword(@"\name"))
{
primaryName = RdText.GetText(context);
}
else
{
context.Tokens.Advance(2);
}
}
else
{
context.Tokens.MoveToNextToken();
}
}
// Merge descriptions into signatures
if (argumentDescriptions != null && signatureInfos != null)
{
foreach (ISignatureInfo sigInfo in signatureInfos)
{
// Add missing arguments from the \arguments{} section
foreach (var arg in sigInfo.Arguments)
{
string description;
if (argumentDescriptions.TryGetValue(arg.Name, out description))
{
((NamedItemInfo)arg).Description = description ?? string.Empty;
}
}
}
}
// Merge signatures into function infos
var functionInfos = new Dictionary <string, FunctionInfo>();
if (signatureInfos != null)
{
var functionSignatures = new Dictionary <string, List <ISignatureInfo> >();
foreach (ISignatureInfo sigInfo in signatureInfos)
{
FunctionInfo functionInfo;
List <ISignatureInfo> sigList;
if (!functionInfos.TryGetValue(sigInfo.FunctionName, out functionInfo))
{
// Create function info
functionInfo = CreateFunctionInfo(sigInfo.FunctionName, functionDescription, returnValue, isInternal);
functionInfos[sigInfo.FunctionName] = functionInfo;
// Create list of signatures for this function
sigList = new List <ISignatureInfo>();
functionSignatures[sigInfo.FunctionName] = sigList;
functionInfo.Signatures = sigList;
}
else
{
sigList = functionSignatures[sigInfo.FunctionName];
}
sigList.Add(sigInfo);
}
}
// Propage to aliases
if (!string.IsNullOrWhiteSpace(primaryName))
{
FunctionInfo functionInfo;
if (functionInfos.TryGetValue(primaryName, out functionInfo))
{
foreach (var alias in aliases)
{
if (!functionInfos.ContainsKey(alias))
{
functionInfos[alias] = new FunctionInfo(alias, functionInfo);
}
}
}
}
return(functionInfos.Values.ToList());
}
