public override CommandResult Invoke(Guid group, int id, object inputArg, ref object outputArg)
{
string originalText = TargetBuffer.CurrentSnapshot.GetText();
string formattedText = string.Empty;
var formatter = new RFormatter(REditorSettings.FormatOptions);
try {
formattedText = formatter.Format(originalText);
} catch (Exception ex) {
Debug.Assert(false, "Formatter exception: ", ex.Message);
}
if (!string.IsNullOrEmpty(formattedText) && !string.Equals(formattedText, originalText, StringComparison.Ordinal))
{
var selectionTracker = new RSelectionTracker(TextView, TargetBuffer, new TextRange(0, TargetBuffer.CurrentSnapshot.Length));
selectionTracker.StartTracking(automaticTracking: false);
try {
using (var massiveChange = new MassiveChange(TextView, TargetBuffer, EditorShell, Resources.FormatDocument)) {
IREditorDocument document = REditorDocument.TryFromTextBuffer(TargetBuffer);
if (document != null)
{
document.EditorTree.Invalidate();
}
var caretPosition = TextView.Caret.Position.BufferPosition;
var viewPortLeft = TextView.ViewportLeft;
RTokenizer tokenizer = new RTokenizer();
string oldText = TargetBuffer.CurrentSnapshot.GetText();
IReadOnlyTextRangeCollection <RToken> oldTokens = tokenizer.Tokenize(oldText);
IReadOnlyTextRangeCollection <RToken> newTokens = tokenizer.Tokenize(formattedText);
#if DEBUG
//if (oldTokens.Count != newTokens.Count) {
// for (int i = 0; i < Math.Min(oldTokens.Count, newTokens.Count); i++) {
// if (oldTokens[i].TokenType != newTokens[i].TokenType) {
// Debug.Assert(false, Invariant($"Token type difference at {i}"));
// break;
// } else if (oldTokens[i].Length != newTokens[i].Length) {
// Debug.Assert(false, Invariant($"token length difference at {i}"));
// break;
// }
// }
//}
#endif
IncrementalTextChangeApplication.ApplyChangeByTokens(
TargetBuffer,
new TextStream(oldText), new TextStream(formattedText),
oldTokens, newTokens,
TextRange.FromBounds(0, oldText.Length),
Resources.FormatDocument, selectionTracker, EditorShell);
}
} finally {
selectionTracker.EndTracking();
}
return(new CommandResult(CommandStatus.Supported, 0));
}
return(CommandResult.NotSupported);
}
private void TokenFromPosition(ITextSnapshot snapshot, int position, out int itemIndex, out int offset)
{
// Normally token stream does not change after formatting so we can simply rely on the fact
// that caret position is going to remain relative to the same token index
itemIndex = -1;
offset = 0;
// Expand range to include the next line. This is needed when user introduces line break.
var lineNumber = snapshot.GetLineNumberFromPosition(_changingRange.End);
if (lineNumber < snapshot.LineCount - 1)
{
var end = snapshot.GetLineFromLineNumber(lineNumber + 1).End;
_changingRange = TextRange.FromBounds(_changingRange.Start, end);
}
var tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection <RToken> tokens =
tokenizer.Tokenize(new TextProvider(snapshot), _changingRange.Start, _changingRange.Length, true);
// Check if position is adjacent to previous token
int prevItemIndex = tokens.GetFirstItemBeforePosition(position);
if (prevItemIndex >= 0 && tokens[prevItemIndex].End == position)
{
itemIndex = prevItemIndex;
offset = -tokens[itemIndex].Length;
return;
}
int nextItemIndex = tokens.GetFirstItemAfterOrAtPosition(position);
if (nextItemIndex >= 0)
{
// If two tokens are adjacent, gravity is negative, i.e. caret travels
// with preceding token so it won't just to aniother line if, say,
// formatter decides to insert a new line between tokens.
if (nextItemIndex > 0 && tokens[nextItemIndex - 1].End == tokens[nextItemIndex].Start)
{
nextItemIndex--;
}
offset = tokens[nextItemIndex].Start - position;
itemIndex = nextItemIndex;
return;
}
// We are past last token
if (tokens.Count > 0)
{
itemIndex = tokens.Count - 1;
offset = tokens[itemIndex].Start - position;
}
else
{
itemIndex = -1;
offset = position;
}
}
public override bool Parse(ParseContext context, IAstNode parent)
{
RToken currentToken = context.Tokens.CurrentToken;
string text = context.TextProvider.GetText(currentToken);
double realPart = 0;
double imaginaryPart = 0;
Debug.Assert(currentToken.TokenType == RTokenType.Complex);
// Split into real and imaginary parts. Imaginary part
// should always be there since otherwise tokenizer would
// not have idenfified the number as complex. Note that
// real part may be missing as in '+0i'. Operator may also
// be missing: 1i is a legal complex number.
Debug.Assert(text[text.Length - 1] == 'i');
// Drop trailing i and retokenize as two numbers
RTokenizer tokenizer = new RTokenizer(separateComments: false);
IReadOnlyTextRangeCollection <RToken> tokens = tokenizer.Tokenize(text.Substring(0, text.Length - 1));
if (tokens.Count == 1)
{
// Only imaginary part is present
Debug.Assert(tokens[0].TokenType == RTokenType.Number);
// TODO: handle complex numbers in Hex
if (!Double.TryParse(text.Substring(tokens[0].Start, tokens[0].Length), out imaginaryPart))
{
imaginaryPart = 0;
}
}
else if (tokens.Count == 3)
{
// Real and imaginary parts present
Debug.Assert(tokens[0].TokenType == RTokenType.Number);
Debug.Assert(tokens[1].TokenType == RTokenType.Operator);
Debug.Assert(tokens[2].TokenType == RTokenType.Number);
// TODO: handle complex numbers in Hex
if (!Double.TryParse(text.Substring(tokens[0].Start, tokens[0].Length), out realPart))
{
realPart = 0;
}
if (!Double.TryParse(text.Substring(tokens[2].Start, tokens[2].Length), out imaginaryPart))
{
imaginaryPart = 0;
}
}
else
{
context.AddError(new MissingItemParseError(ParseErrorType.NumberExpected, context.Tokens.PreviousToken));
return(false);
}
Complex complex = new Complex(realPart, imaginaryPart);
NodeValue = new RComplex(complex);
return(base.Parse(context, parent));
}
public static AstRoot Parse(ITextProvider textProvider, ITextRange range, IExpressionTermFilter filter)
{
var tokenizer = new RTokenizer(separateComments: true);
IReadOnlyTextRangeCollection <RToken> tokens = tokenizer.Tokenize(textProvider, range.Start, range.Length);
TokenStream <RToken> tokenStream = new TokenStream <RToken>(tokens, new RToken(RTokenType.EndOfStream, TextRange.EmptyRange));
return(Parse(textProvider, range, tokenStream, tokenizer.CommentTokens, filter));
}
internal static bool IsPackageListCompletion(ITextBuffer textBuffer, int position)
{
ITextSnapshot snapshot = textBuffer.CurrentSnapshot;
ITextSnapshotLine line = snapshot.GetLineFromPosition(position);
string lineText = line.GetText();
int linePosition = position - line.Start;
// We should be either at library(| or inside library(|)
// or over package name like in library(ba|se)
// Go left and right looking for
RTokenizer tokenizer = new RTokenizer();
ITextProvider textProvider = new TextStream(lineText);
IReadOnlyTextRangeCollection <RToken> c = tokenizer.Tokenize(textProvider, 0, textProvider.Length);
TokenStream <RToken> tokens = new TokenStream <RToken>(c, RToken.EndOfStreamToken);
while (!tokens.IsEndOfStream())
{
if (tokens.CurrentToken.Start >= linePosition)
{
break;
}
if (tokens.CurrentToken.TokenType == RTokenType.Identifier)
{
string identifier = textProvider.GetText(tokens.CurrentToken);
if (identifier == "library" || identifier == "require")
{
tokens.MoveToNextToken();
if (tokens.CurrentToken.TokenType == RTokenType.OpenBrace)
{
RToken openBrace = tokens.CurrentToken;
while (!tokens.IsEndOfStream())
{
if (tokens.CurrentToken.TokenType == RTokenType.CloseBrace)
{
if (linePosition >= openBrace.End && linePosition <= tokens.CurrentToken.Start)
{
return(true);
}
return(false);
}
else if (tokens.NextToken.TokenType == RTokenType.EndOfStream)
{
return(true);
}
tokens.MoveToNextToken();
}
}
}
}
tokens.MoveToNextToken();
}
return(false);
}
public TokenStream(IReadOnlyTextRangeCollection <T> tokens, T endOfStreamToken)
{
Check.ArgumentNull(nameof(tokens), tokens);
_index = 0;
_tokens = tokens;
_endOfStreamToken = endOfStreamToken;
_isEndOfStream = tokens.Length == 0;
CurrentToken = _isEndOfStream ? _endOfStreamToken : _tokens[0];
}
public void Tokenize_BuiltIns01()
{
IReadOnlyTextRangeCollection <RToken> tokens = this.Tokenize("require library switch return", new RTokenizer());
tokens.Should().HaveCount(4);
foreach (var token in tokens)
{
token.Should().HaveType(RTokenType.Identifier).And.HaveSubType(RTokenSubType.BuiltinFunction);
}
}
/// <summary>
/// Given RD data and function name parses the data and creates structured
/// information about the function. Method returns multiple functions since
/// RD data often provides information on several functions so in order
/// to avoid processing same data multiple times parser extracts information
/// on all related functions.
/// </summary>
public static IReadOnlyList <IFunctionInfo> GetFunctionInfos(string rdHelpData)
{
var tokenizer = new RdTokenizer(tokenizeRContent: false);
ITextProvider textProvider = new TextStream(rdHelpData);
IReadOnlyTextRangeCollection <RdToken> tokens = tokenizer.Tokenize(textProvider, 0, textProvider.Length);
RdParseContext context = new RdParseContext(tokens, textProvider);
return(ParseFunctions(context));
}
public void Tokenize_Missing()
{
string s = "NA NA_character_ NA_complex_ NA_integer_ NA_real_";
IReadOnlyTextRangeCollection <RToken> tokens = this.Tokenize(s, new RTokenizer());
tokens.Should().HaveCount(5);
foreach (var token in tokens)
{
token.Should().HaveType(RTokenType.Missing).And.HaveSubType(RTokenSubType.BuiltinConstant);
}
}
public void Tokenize_BuiltIns02()
{
IReadOnlyTextRangeCollection <RToken> tokens = this.Tokenize("require() library() switch() return()", new RTokenizer());
tokens.Should().HaveCount(12);
for (var i = 0; i < tokens.Count; i += 3)
{
tokens[i].Should().HaveType(RTokenType.Identifier).And.HaveSubType(RTokenSubType.BuiltinFunction);
tokens[i + 1].Should().HaveType(RTokenType.OpenBrace);
tokens[i + 2].Should().HaveType(RTokenType.CloseBrace);
}
}
private static bool IsMultiLineCandidate(string text)
{
if (text.IndexOfAny(new[] { '\n', '\r' }) != -1)
{
// if we already have newlines then we're multiline
return(true);
}
var tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection <RToken> tokens = tokenizer.Tokenize(new TextStream(text), 0, text.Length);
return(tokens.Any(t => t.TokenType == RTokenType.OpenCurlyBrace));
}
public TokenStream(IReadOnlyTextRangeCollection <T> tokens, T endOfStreamToken)
{
if (tokens == null)
{
throw new ArgumentNullException("tokens");
}
_index = 0;
_tokens = tokens;
_endOfStreamToken = endOfStreamToken;
_isEndOfStream = tokens.Length == 0;
_currentToken = _isEndOfStream ? _endOfStreamToken : _tokens[0];
}
private static void TokenFromPosition(ITextSnapshot snapshot, int position, out int itemIndex, out int offset)
{
// Normally token stream does not change after formatting so we can simply rely on the fact
// that caret position is going to remain relative to the same token index
itemIndex = -1;
offset = 0;
var tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection <RToken> tokens = tokenizer.Tokenize(new TextProvider(snapshot), 0, snapshot.Length, true);
// Check if position is adjacent to previous token
int prevItemIndex = tokens.GetFirstItemBeforePosition(position);
if (prevItemIndex >= 0 && tokens[prevItemIndex].End == position)
{
itemIndex = prevItemIndex;
offset = -tokens[itemIndex].Length;
return;
}
int nextItemIndex = tokens.GetFirstItemAfterOrAtPosition(position);
if (nextItemIndex >= 0)
{
// If two tokens are adjacent, gravity is negative, i.e. caret travels
// with preceding token so it won't just to aniother line if, say,
// formatter decides to insert a new line between tokens.
if (nextItemIndex > 0 && tokens[nextItemIndex - 1].End == tokens[nextItemIndex].Start)
{
nextItemIndex--;
}
offset = tokens[nextItemIndex].Start - position;
itemIndex = nextItemIndex;
return;
}
// We are past last token
if (tokens.Count > 0)
{
itemIndex = tokens.Count - 1;
offset = tokens[itemIndex].Start - position;
}
else
{
itemIndex = -1;
offset = position;
}
}
/// <summary>
/// Parse text from a text provider within a given range
/// </summary>
/// <param name="textProvider">Text provider</param>
/// <param name="range">Range to parse</param>
public static AstRoot Parse(ITextProvider textProvider, ITextRange range)
{
var tokenizer = new RTokenizer(separateComments: true);
IReadOnlyTextRangeCollection <RToken> tokens = tokenizer.Tokenize(textProvider, range.Start, range.Length);
TokenStream <RToken> tokenStream = new TokenStream <RToken>(tokens, new RToken(RTokenType.EndOfStream, TextRange.EmptyRange));
ParseContext context = new ParseContext(textProvider, range, tokenStream, tokenizer.CommentTokens);
context.AstRoot.Parse(context, context.AstRoot);
context.AstRoot.Errors = new TextRangeCollection <IParseError>(context.Errors);
return(context.AstRoot);
}
public static IReadOnlyList <ISignatureInfo> ParseSignatures(string usageContent, IReadOnlyDictionary <string, string> argumentsDescriptions = null)
{
// RD signature text may contain \dots sequence which denotes ellipsis.
// R parser does not know about it and hence we will replace \dots by ...
// Also, signatures may contain S3 method info like
// '\method{as.matrix}{data.frame}(x, rownames.force = NA, \dots)'
// which we need to filter out since they are irrelevant to intellisense.
List <ISignatureInfo> signatures = new List <ISignatureInfo>();
usageContent = usageContent.Replace(@"\dots", "...");
RTokenizer tokenizer = new RTokenizer(separateComments: true);
IReadOnlyTextRangeCollection <RToken> collection = tokenizer.Tokenize(usageContent);
ITextProvider textProvider = new TextStream(usageContent);
TokenStream <RToken> tokens = new TokenStream <RToken>(collection, RToken.EndOfStreamToken);
var parseContext = new ParseContext(textProvider,
TextRange.FromBounds(tokens.CurrentToken.Start, textProvider.Length),
tokens, tokenizer.CommentTokens);
while (!tokens.IsEndOfStream())
{
// Filter out '\method{...}{}(signature)
if (tokens.CurrentToken.TokenType == RTokenType.OpenCurlyBrace)
{
// Check if { is preceded by \method
}
if (tokens.CurrentToken.TokenType != RTokenType.Identifier)
{
break;
}
string functionName = textProvider.GetText(tokens.CurrentToken);
tokens.MoveToNextToken();
ISignatureInfo info = ParseSignature(functionName, parseContext, argumentsDescriptions);
if (info != null)
{
signatures.Add(info);
}
}
return(signatures);
}
public static bool FormatRangeExact(ITextView textView, ITextBuffer textBuffer,
ITextRange formatRange, RFormatOptions options)
{
ITextSnapshot snapshot = textBuffer.CurrentSnapshot;
Span spanToFormat = new Span(formatRange.Start, formatRange.Length);
string spanText = snapshot.GetText(spanToFormat.Start, spanToFormat.Length);
string trimmedSpanText = spanText.Trim();
RFormatter formatter = new RFormatter(options);
string formattedText = formatter.Format(trimmedSpanText);
formattedText = formattedText.Trim(); // There may be inserted line breaks after {
// Apply formatted text without indentation. We then will update the parse tree
// so we can calculate proper line indents from the AST via the smart indenter.
if (!spanText.Equals(formattedText, StringComparison.Ordinal))
{
// Extract existing indent before applying changes. Existing indent
// may be used by the smart indenter for function argument lists.
var startLine = snapshot.GetLineFromPosition(spanToFormat.Start);
var originalIndentSizeInSpaces = IndentBuilder.TextIndentInSpaces(startLine.GetText(), options.IndentSize);
var selectionTracker = new RSelectionTracker(textView, textBuffer, formatRange);
RTokenizer tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection <RToken> oldTokens = tokenizer.Tokenize(spanText);
IReadOnlyTextRangeCollection <RToken> newTokens = tokenizer.Tokenize(formattedText);
IncrementalTextChangeApplication.ApplyChangeByTokens(
textBuffer,
new TextStream(spanText), new TextStream(formattedText),
oldTokens, newTokens,
formatRange,
Resources.AutoFormat, selectionTracker,
() => {
var ast = UpdateAst(textBuffer);
// Apply indentation
IndentLines(textView, textBuffer, new TextRange(formatRange.Start, formattedText.Length), ast, options, originalIndentSizeInSpaces);
});
return(true);
}
return(false);
}
public static bool FormatRangeExact(ITextView textView, ITextBuffer textBuffer, ITextRange formatRange,
AstRoot ast, RFormatOptions options,
int scopeStatementPosition, bool respectUserIndent = true)
{
ITextSnapshot snapshot = textBuffer.CurrentSnapshot;
Span spanToFormat = new Span(formatRange.Start, formatRange.Length);
string spanText = snapshot.GetText(spanToFormat.Start, spanToFormat.Length);
string trimmedSpanText = spanText.Trim();
if (trimmedSpanText == "}")
{
// Locate opening { and its statement
var scopeNode = ast.GetNodeOfTypeFromPosition <IAstNodeWithScope>(spanToFormat.Start);
if (scopeNode != null)
{
scopeStatementPosition = scopeNode.Start;
}
}
RFormatter formatter = new RFormatter(options);
string formattedText = formatter.Format(trimmedSpanText);
formattedText = formattedText.Trim(); // there may be inserted line breaks after {
formattedText = IndentLines(textBuffer, spanToFormat.Start, ast, formattedText, options, scopeStatementPosition, respectUserIndent);
if (!spanText.Equals(formattedText, StringComparison.Ordinal))
{
var selectionTracker = new RSelectionTracker(textView, textBuffer);
RTokenizer tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection <RToken> oldTokens = tokenizer.Tokenize(spanText);
IReadOnlyTextRangeCollection <RToken> newTokens = tokenizer.Tokenize(formattedText);
IncrementalTextChangeApplication.ApplyChangeByTokens(
textBuffer,
new TextStream(spanText), new TextStream(formattedText),
oldTokens, newTokens,
formatRange,
Resources.AutoFormat, selectionTracker);
return(true);
}
return(false);
}
public static string WriteTokens <Token, TokenType>(IReadOnlyTextRangeCollection <Token> tokens) where Token : ITextRange
{
var sb = new StringBuilder();
foreach (Token t in tokens)
{
if (t is ICompositeToken)
{
WriteCompositeToken(t as ICompositeToken, sb);
}
else
{
WriteToken <Token, TokenType>(t, sb);
}
}
string s = sb.ToString();
return(s);
}
public ErrorArgument(IEnumerable <RToken> tokens)
{
Tokens = new TextRangeCollection <RToken>(tokens);
}
public RdParseContext(string packageName, IReadOnlyTextRangeCollection <RdToken> tokens, ITextProvider textProvider)
{
PackageName = packageName;
TextProvider = textProvider;
Tokens = new TokenStream <RdToken>(tokens, RdToken.EndOfStreamToken);
}
public RdParseContext(IReadOnlyTextRangeCollection <RdToken> tokens, ITextProvider textProvider)
{
this.TextProvider = textProvider;
this.Tokens = new TokenStream <RdToken>(tokens, RdToken.EndOfStreamToken);
}
public RdParseContext(IReadOnlyTextRangeCollection<RdToken> tokens, ITextProvider textProvider) {
this.TextProvider = textProvider;
this.Tokens = new TokenStream<RdToken>(tokens, RdToken.EndOfStreamToken);
}
public ErrorArgument(IEnumerable<RToken> tokens) {
Tokens = new TextRangeCollection<RToken>(tokens);
}
public virtual bool GetLanguageBracesFromPosition(
BraceType braceType,
int position, bool reversed, out int start, out int end)
{
TokenTypeT startTokenType = BraceTypeToTokenTypeMap[braceType].OpenBrace;
TokenTypeT endTokenType = BraceTypeToTokenTypeMap[braceType].CloseBrace;
IReadOnlyTextRangeCollection <TokenClassT> tokens = GetTokens(0, TextBuffer.CurrentSnapshot.Length);
start = -1;
end = -1;
Stack <TokenTypeT> stack = new Stack <TokenTypeT>();
int startIndex = -1;
for (int i = 0; i < tokens.Count; i++)
{
if (tokens[i].Start == position)
{
startIndex = i;
break;
}
}
if (startIndex < 0)
{
return(false);
}
if (_tokenComparer.Compare(tokens[startIndex].TokenType, startTokenType) != 0 && _tokenComparer.Compare(tokens[startIndex].TokenType, endTokenType) != 0)
{
return(false);
}
if (!reversed)
{
for (int i = startIndex; i < tokens.Count; i++)
{
TokenClassT token = tokens[i];
if (token.TokenType.Equals(startTokenType))
{
stack.Push(token.TokenType);
}
else if (_tokenComparer.Compare(token.TokenType, endTokenType) == 0)
{
if (stack.Count > 0)
{
stack.Pop();
}
if (stack.Count == 0)
{
start = tokens[startIndex].Start;
end = token.Start;
return(true);
}
}
}
}
else
{
for (int i = startIndex; i >= 0; i--)
{
TokenClassT token = tokens[i];
if (_tokenComparer.Compare(token.TokenType, endTokenType) == 0)
{
stack.Push(token.TokenType);
}
else if (_tokenComparer.Compare(token.TokenType, startTokenType) == 0)
{
if (stack.Count > 0)
{
stack.Pop();
}
if (stack.Count == 0)
{
start = token.Start;
end = token.Start;
end = tokens[startIndex].Start;
return(true);
}
}
}
}
return(false);
}
