public static AstRoot Parse(ITextProvider textProvider, ITextRange range, IExpressionTermFilter filter) {
var tokenizer = new RTokenizer(separateComments: true);
IReadOnlyTextRangeCollection<RToken> tokens = tokenizer.Tokenize(textProvider, range.Start, range.Length);
TokenStream<RToken> tokenStream = new TokenStream<RToken>(tokens, new RToken(RTokenType.EndOfStream, TextRange.EmptyRange));
return Parse(textProvider, range, tokenStream, tokenizer.CommentTokens, filter);
}
public override CommandResult Invoke(Guid group, int id, object inputArg, ref object outputArg) {
string originalText = TargetBuffer.CurrentSnapshot.GetText();
string formattedText = string.Empty;
var formatter = new RFormatter(REditorSettings.FormatOptions);
try {
formattedText = formatter.Format(originalText);
} catch (Exception ex) {
Debug.Assert(false, "Formatter exception: ", ex.Message);
}
if (!string.IsNullOrEmpty(formattedText) && !string.Equals(formattedText, originalText, StringComparison.Ordinal)) {
var selectionTracker = new RSelectionTracker(TextView, TargetBuffer, new TextRange(0, TargetBuffer.CurrentSnapshot.Length));
selectionTracker.StartTracking(automaticTracking: false);
try {
using (var massiveChange = new MassiveChange(TextView, TargetBuffer, EditorShell, Resources.FormatDocument)) {
IREditorDocument document = REditorDocument.TryFromTextBuffer(TargetBuffer);
if (document != null) {
document.EditorTree.Invalidate();
}
var caretPosition = TextView.Caret.Position.BufferPosition;
var viewPortLeft = TextView.ViewportLeft;
RTokenizer tokenizer = new RTokenizer();
string oldText = TargetBuffer.CurrentSnapshot.GetText();
IReadOnlyTextRangeCollection<RToken> oldTokens = tokenizer.Tokenize(oldText);
IReadOnlyTextRangeCollection<RToken> newTokens = tokenizer.Tokenize(formattedText);
#if DEBUG
//if (oldTokens.Count != newTokens.Count) {
// for (int i = 0; i < Math.Min(oldTokens.Count, newTokens.Count); i++) {
// if (oldTokens[i].TokenType != newTokens[i].TokenType) {
// Debug.Assert(false, Invariant($"Token type difference at {i}"));
// break;
// } else if (oldTokens[i].Length != newTokens[i].Length) {
// Debug.Assert(false, Invariant($"token length difference at {i}"));
// break;
// }
// }
//}
#endif
IncrementalTextChangeApplication.ApplyChangeByTokens(
TargetBuffer,
new TextStream(oldText), new TextStream(formattedText),
oldTokens, newTokens,
TextRange.FromBounds(0, oldText.Length),
Resources.FormatDocument, selectionTracker, EditorShell);
}
} finally {
selectionTracker.EndTracking();
}
return new CommandResult(CommandStatus.Supported, 0);
}
return CommandResult.NotSupported;
}
public static string BacktickName(string name) {
if (!string.IsNullOrEmpty(name)) {
var t = new RTokenizer();
var tokens = t.Tokenize(name);
if (tokens.Count > 1) {
return Invariant($"`{name}`");
}
}
return name;
}
/// <summary>
/// Parse text from a text provider within a given range
/// </summary>
/// <param name="textProvider">Text provider</param>
/// <param name="range">Range to parse</param>
public static AstRoot Parse(ITextProvider textProvider, ITextRange range) {
var tokenizer = new RTokenizer(separateComments: true);
IReadOnlyTextRangeCollection<RToken> tokens = tokenizer.Tokenize(textProvider, range.Start, range.Length);
TokenStream<RToken> tokenStream = new TokenStream<RToken>(tokens, new RToken(RTokenType.EndOfStream, TextRange.EmptyRange));
ParseContext context = new ParseContext(textProvider, range, tokenStream, tokenizer.CommentTokens);
context.AstRoot.Parse(context, context.AstRoot);
context.AstRoot.Errors = new TextRangeCollection<IParseError>(context.Errors);
return context.AstRoot;
}
public static IReadOnlyList <ISignatureInfo> ParseSignatures(string usageContent, IReadOnlyDictionary <string, string> argumentsDescriptions = null)
{
// RD signature text may contain \dots sequence which denotes ellipsis.
// R parser does not know about it and hence we will replace \dots by ...
// Also, signatures may contain S3 method info like
// '\method{as.matrix}{data.frame}(x, rownames.force = NA, \dots)'
// which we need to filter out since they are irrelevant to intellisense.
List <ISignatureInfo> signatures = new List <ISignatureInfo>();
usageContent = usageContent.Replace(@"\dots", "...");
RTokenizer tokenizer = new RTokenizer(separateComments: true);
IReadOnlyTextRangeCollection <RToken> collection = tokenizer.Tokenize(usageContent);
ITextProvider textProvider = new TextStream(usageContent);
TokenStream <RToken> tokens = new TokenStream <RToken>(collection, RToken.EndOfStreamToken);
var parseContext = new ParseContext(textProvider,
TextRange.FromBounds(tokens.CurrentToken.Start, textProvider.Length),
tokens, tokenizer.CommentTokens);
while (!tokens.IsEndOfStream())
{
// Filter out '\method{...}{}(signature)
if (tokens.CurrentToken.TokenType == RTokenType.OpenCurlyBrace)
{
// Check if { is preceded by \method
}
if (tokens.CurrentToken.TokenType != RTokenType.Identifier)
{
break;
}
string functionName = textProvider.GetText(tokens.CurrentToken);
tokens.MoveToNextToken();
ISignatureInfo info = ParseSignature(functionName, parseContext, argumentsDescriptions);
if (info != null)
{
signatures.Add(info);
}
}
return(signatures);
}
/// <summary>
/// Extracts complete variable name under the caret. In '`abc`$`def` returns
/// the complete expression rather than its parts. Typically used to get data
/// for completion of variable members as in when user typed 'abc$def$'
/// Since method does not perform semantic analysis, it does not guaratee
/// syntactically correct expression, it may return 'a$$b'.
/// </summary>
public static string GetVariableNameBeforeCaret(this ITextView textView)
{
if (textView.Caret.InVirtualSpace)
{
return(string.Empty);
}
var position = textView.Caret.Position.BufferPosition;
var line = position.GetContainingLine();
// For performance reasons we won't be using AST here
// since during completion it is most probably damaged.
string lineText = line.GetText();
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(lineText);
var tokenPosition = position - line.Start;
var index = tokens.GetFirstItemBeforePosition(tokenPosition);
// Preceding token must be right next to caret position
if (index < 0 || tokens[index].End < tokenPosition || !IsVariableNameToken(lineText, tokens[index]))
{
return(string.Empty);
}
if (index == 0)
{
return(IsVariableNameToken(lineText, tokens[0]) ? lineText.Substring(tokens[0].Start, tokens[0].Length) : string.Empty);
}
// Walk back through tokens allowing identifier and specific
// operator tokens. No whitespace is permitted between tokens.
// We have at least 2 tokens here.
int i = index;
for (; i > 0; i--)
{
var precedingToken = tokens[i - 1];
var currentToken = tokens[i];
if (precedingToken.End < currentToken.Start || !IsVariableNameToken(lineText, precedingToken))
{
break;
}
}
return(lineText.Substring(tokens[i].Start, tokens[index].End - tokens[i].Start));
}
/// <summary>
/// Called before character type is passed down to the core editor
/// along the controll chain. Gives language-specific controller
/// a chance to initiate different action and potentially 'eat'
/// the character. For example, in R typing 'abc[TAB] should bring
/// up intellisense list rather than actually insert the tab character.
/// </summary>
/// <returns>
/// True if character was handled and should not be
/// passed down to core editor or false otherwise.
/// </returns>
public override bool OnPreTypeChar(char typedCharacter)
{
// Allow tab to bring intellisense if
// a) REditorSettings.ShowCompletionOnTab true
// b) Position is at the end of a string so we bring completion for files
// c) There is no selection
if (typedCharacter == '\t' && !HasActiveCompletionSession && TextView.Selection.StreamSelectionSpan.Length == 0)
{
// if previous character is identifier character, bring completion list
SnapshotPoint?position = REditorDocument.MapCaretPositionFromView(TextView);
if (position.HasValue)
{
int pos = position.Value;
var doc = REditorDocument.FromTextBuffer(position.Value.Snapshot.TextBuffer);
if (!doc.IsPositionInComment(pos))
{
if (pos > 0 && pos <= position.Value.Snapshot.Length)
{
bool endOfIdentifier = RTokenizer.IsIdentifierCharacter(position.Value.Snapshot[pos - 1]);
bool showCompletion = endOfIdentifier && _settings.ShowCompletionOnTab;
if (!showCompletion)
{
var document = REditorDocument.FromTextBuffer(position.Value.Snapshot.TextBuffer);
string directory;
showCompletion = RCompletionEngine.CanShowFileCompletion(document.EditorTree.AstRoot, pos, out directory);
}
if (showCompletion)
{
ShowCompletion(autoShownCompletion: false);
return(true); // eat the character
}
}
}
}
}
else if (typedCharacter == '#')
{
if (TryInsertRoxygenBlock())
{
return(true);
}
}
return(base.OnPreTypeChar(typedCharacter));
}
/// <summary>
/// Calculates span in the text buffer that contains data
/// applicable to the current completion session. A tracking
/// span will be created over it and editor will grow and shrink
/// tracking span as user types and filter completion session
/// based on the data inside the tracking span.
/// </summary>
private Span GetApplicableSpan(int position, ICompletionSession session)
{
var selectedSpans = session.TextView.Selection.SelectedSpans;
if (selectedSpans.Count == 1 && selectedSpans[0].Span.Length > 0)
{
return(selectedSpans[0].Span);
}
ITextSnapshot snapshot = _textBuffer.CurrentSnapshot;
ITextSnapshotLine line = snapshot.GetLineFromPosition(position);
string lineText = snapshot.GetText(line.Start, line.Length);
int linePosition = position - line.Start;
int start = 0;
int end = line.Length;
for (int i = linePosition - 1; i >= 0; i--)
{
char ch = lineText[i];
if (!RTokenizer.IsIdentifierCharacter(ch))
{
start = i + 1;
break;
}
}
for (int i = linePosition; i < lineText.Length; i++)
{
char ch = lineText[i];
if (!RTokenizer.IsIdentifierCharacter(ch))
{
end = i;
break;
}
}
if (start < end)
{
return(new Span(start + line.Start, end - start));
}
return(new Span(position, 0));
}
public static void TokenizeFileImplementation(CoreTestFilesFixture fixture, string name) {
string testFile = fixture.GetDestinationPath(name);
string baselineFile = testFile + ".tokens";
string text = fixture.LoadDestinationFile(name);
ITextProvider textProvider = new TextStream(text);
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(textProvider, 0, textProvider.Length);
string actual = DebugWriter.WriteTokens<RToken, RTokenType>(tokens);
if (_regenerateBaselineFiles) {
// Update this to your actual enlistment if you need to update baseline
baselineFile = Path.Combine(fixture.SourcePath, @"Tokenization\", Path.GetFileName(testFile)) + ".tokens";
TestFiles.UpdateBaseline(baselineFile, actual);
} else {
TestFiles.CompareToBaseLine(baselineFile, actual);
}
}
public static bool FormatRangeExact(ITextView textView, ITextBuffer textBuffer,
ITextRange formatRange, RFormatOptions options)
{
ITextSnapshot snapshot = textBuffer.CurrentSnapshot;
Span spanToFormat = new Span(formatRange.Start, formatRange.Length);
string spanText = snapshot.GetText(spanToFormat.Start, spanToFormat.Length);
string trimmedSpanText = spanText.Trim();
RFormatter formatter = new RFormatter(options);
string formattedText = formatter.Format(trimmedSpanText);
formattedText = formattedText.Trim(); // There may be inserted line breaks after {
// Apply formatted text without indentation. We then will update the parse tree
// so we can calculate proper line indents from the AST via the smart indenter.
if (!spanText.Equals(formattedText, StringComparison.Ordinal))
{
// Extract existing indent before applying changes. Existing indent
// may be used by the smart indenter for function argument lists.
var startLine = snapshot.GetLineFromPosition(spanToFormat.Start);
var originalIndentSizeInSpaces = IndentBuilder.TextIndentInSpaces(startLine.GetText(), options.IndentSize);
var selectionTracker = new RSelectionTracker(textView, textBuffer, formatRange);
RTokenizer tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection <RToken> oldTokens = tokenizer.Tokenize(spanText);
IReadOnlyTextRangeCollection <RToken> newTokens = tokenizer.Tokenize(formattedText);
IncrementalTextChangeApplication.ApplyChangeByTokens(
textBuffer,
new TextStream(spanText), new TextStream(formattedText),
oldTokens, newTokens,
formatRange,
Resources.AutoFormat, selectionTracker,
() => {
var ast = UpdateAst(textBuffer);
// Apply indentation
IndentLines(textView, textBuffer, new TextRange(formatRange.Start, formattedText.Length), ast, options, originalIndentSizeInSpaces);
});
return(true);
}
return(false);
}
private int PositionFromTokens(ITextSnapshot snapshot, int itemIndex, int offset)
{
var lengthChange = snapshot.Length - _lengthBeforeChange;
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(
new TextProvider(snapshot), _changingRange.Start, _changingRange.Length + lengthChange, true);
if (itemIndex >= 0 && itemIndex < tokens.Count)
{
var position = tokens[itemIndex].Start - offset;
position = Math.Min(position, snapshot.Length);
position = Math.Max(position, 0);
return(position);
}
return(_changingRange.End + lengthChange);
}
public static bool FormatRangeExact(ITextView textView, ITextBuffer textBuffer, ITextRange formatRange,
AstRoot ast, RFormatOptions options,
int scopeStatementPosition, bool respectUserIndent = true)
{
ITextSnapshot snapshot = textBuffer.CurrentSnapshot;
Span spanToFormat = new Span(formatRange.Start, formatRange.Length);
string spanText = snapshot.GetText(spanToFormat.Start, spanToFormat.Length);
string trimmedSpanText = spanText.Trim();
if (trimmedSpanText == "}")
{
// Locate opening { and its statement
var scopeNode = ast.GetNodeOfTypeFromPosition <IAstNodeWithScope>(spanToFormat.Start);
if (scopeNode != null)
{
scopeStatementPosition = scopeNode.Start;
}
}
RFormatter formatter = new RFormatter(options);
string formattedText = formatter.Format(trimmedSpanText);
formattedText = formattedText.Trim(); // there may be inserted line breaks after {
formattedText = IndentLines(textBuffer, spanToFormat.Start, ast, formattedText, options, scopeStatementPosition, respectUserIndent);
if (!spanText.Equals(formattedText, StringComparison.Ordinal))
{
var selectionTracker = new RSelectionTracker(textView, textBuffer);
RTokenizer tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection <RToken> oldTokens = tokenizer.Tokenize(spanText);
IReadOnlyTextRangeCollection <RToken> newTokens = tokenizer.Tokenize(formattedText);
IncrementalTextChangeApplication.ApplyChangeByTokens(
textBuffer,
new TextStream(spanText), new TextStream(formattedText),
oldTokens, newTokens,
formatRange,
Resources.AutoFormat, selectionTracker);
return(true);
}
return(false);
}
private bool FormatRangeExact(IEditorView editorView, IEditorBuffer editorBuffer, ITextRange formatRange)
{
var snapshot = editorBuffer.CurrentSnapshot;
var spanText = snapshot.GetText(formatRange);
var trimmedSpanText = spanText.Trim();
var formatter = new RFormatter(_settings.FormatOptions);
var formattedText = formatter.Format(trimmedSpanText);
formattedText = formattedText.Trim(); // There may be inserted line breaks after {
// Apply formatted text without indentation. We then will update the parse tree
// so we can calculate proper line indents from the AST via the smart indenter.
if (!spanText.Equals(formattedText, StringComparison.Ordinal))
{
// Extract existing indent before applying changes. Existing indent
// may be used by the smart indenter for function argument lists.
var startLine = snapshot.GetLineFromPosition(formatRange.Start);
var originalIndentSizeInSpaces = IndentBuilder.TextIndentInSpaces(startLine.GetText(), _settings.IndentSize);
var selectionTracker = GetSelectionTracker(editorView, editorBuffer, formatRange);
var tokenizer = new RTokenizer();
var oldTokens = tokenizer.Tokenize(spanText);
var newTokens = tokenizer.Tokenize(formattedText);
var wsChangeHandler = _services.GetService <IIncrementalWhitespaceChangeHandler>();
wsChangeHandler.ApplyChange(
editorBuffer,
new TextStream(spanText), new TextStream(formattedText),
oldTokens, newTokens,
formatRange,
Microsoft.R.Editor.Resources.AutoFormat, selectionTracker,
() => {
var ast = UpdateAst(editorBuffer);
// Apply indentation
IndentLines(editorBuffer, new TextRange(formatRange.Start, formattedText.Length), ast, originalIndentSizeInSpaces);
});
return(true);
}
return(false);
}
private void TokenFromPosition(ITextSnapshot snapshot, int position, out int itemIndex, out int offset) {
// Normally token stream does not change after formatting so we can simply rely on the fact
// that caret position is going to remain relative to the same token index
itemIndex = -1;
offset = 0;
var tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection<RToken> tokens =
tokenizer.Tokenize(new TextProvider(snapshot), _changingRange.Start, _changingRange.Length, true);
// Check if position is adjacent to previous token
int prevItemIndex = tokens.GetFirstItemBeforePosition(position);
if (prevItemIndex >= 0 && tokens[prevItemIndex].End == position) {
itemIndex = prevItemIndex;
offset = -tokens[itemIndex].Length;
return;
}
int nextItemIndex = tokens.GetFirstItemAfterOrAtPosition(position);
if (nextItemIndex >= 0) {
// If two tokens are adjacent, gravity is negative, i.e. caret travels
// with preceding token so it won't just to aniother line if, say,
// formatter decides to insert a new line between tokens.
if (nextItemIndex > 0 && tokens[nextItemIndex - 1].End == tokens[nextItemIndex].Start) {
nextItemIndex--;
}
offset = tokens[nextItemIndex].Start - position;
itemIndex = nextItemIndex;
return;
}
// We are past last token
if (tokens.Count > 0) {
itemIndex = tokens.Count - 1;
offset = tokens[itemIndex].Start - position;
} else {
itemIndex = -1;
offset = position;
}
}
public static string GetVariableName(ITextView textView, ITextSnapshot snapshot)
{
SnapshotPoint?pt = REditorDocument.MapCaretPositionFromView(textView);
if (pt.HasValue && pt.Value > 0)
{
int i = pt.Value - 1;
for (; i >= 0; i--)
{
char ch = snapshot[i];
if (!RTokenizer.IsIdentifierCharacter(ch) && ch != '$' && ch != '@')
{
break;
}
}
return(snapshot.GetText(Span.FromBounds(i + 1, pt.Value)));
}
return(string.Empty);
}
public static Span?GetWordSpan(ITextSnapshot snapshot, int position)
{
ITextSnapshotLine line = snapshot.GetLineFromPosition(position);
if (line.Length == 0)
{
return(null);
}
var text = line.GetText();
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(text);
var positionInLine = position - line.Start;
var token = tokens.FirstOrDefault(t => t.Contains(positionInLine));
if (token != null && token.TokenType != RTokenType.String && token.TokenType != RTokenType.Comment)
{
return(new Span(token.Start + line.Start, token.Length));
}
return(GetWordSpan(text, line.Start, positionInLine));
}
private static IValidationError MultipleStatementsCheck(IAstNode node, LintOptions options, bool projectedBuffer)
{
if (options.MultipleStatements && node is TokenNode t && t.Token.TokenType == RTokenType.Semicolon)
{
var tp = node.Root.TextProvider;
if (!tp.IsNewLineAfterPosition(node.End))
{
// # comment is OK but comments are not part of the AST.
var lineBreakIndex = tp.IndexOf('\n', node.End);
var trailingTextEnd = lineBreakIndex >= 0 ? lineBreakIndex : tp.Length;
var trailingText = tp.GetText(TextRange.FromBounds(node.End, trailingTextEnd));
var tokens = new RTokenizer().Tokenize(trailingText);
var offendingTokens = tokens.Where(x => x.TokenType != RTokenType.Comment);
if (offendingTokens.Any())
{
var squiggle = TextRange.FromBounds(node.End + offendingTokens.First().Start, node.End + offendingTokens.Last().End);
return(new ValidationWarning(squiggle, Resources.Lint_MultipleStatementsInLine, ErrorLocation.Token));
}
}
}
return(null);
}
/// <summary>
/// Determines if position is in object member. Typically used
/// to suppress general intellisense when typing data member
/// name such as 'mtcars$|'
/// </summary>
internal static bool IsInObjectMemberName(ITextProvider textProvider, int position)
{
if (position > 0)
{
for (int i = position - 1; i >= 0; i--)
{
char ch = textProvider[i];
if (ch == '$' || ch == '@')
{
return(true);
}
if (!RTokenizer.IsIdentifierCharacter(ch))
{
break;
}
}
}
return(false);
}
public static bool FormatRangeExact(ITextView textView, ITextBuffer textBuffer, ITextRange formatRange,
AstRoot ast, RFormatOptions options,
int scopeStatementPosition, bool respectUserIndent = true) {
ITextSnapshot snapshot = textBuffer.CurrentSnapshot;
Span spanToFormat = new Span(formatRange.Start, formatRange.Length);
string spanText = snapshot.GetText(spanToFormat.Start, spanToFormat.Length);
string trimmedSpanText = spanText.Trim();
if (trimmedSpanText == "}") {
// Locate opening { and its statement
var scopeNode = ast.GetNodeOfTypeFromPosition<IAstNodeWithScope>(spanToFormat.Start);
if (scopeNode != null) {
scopeStatementPosition = scopeNode.Start;
}
}
RFormatter formatter = new RFormatter(options);
string formattedText = formatter.Format(trimmedSpanText);
formattedText = formattedText.Trim(); // there may be inserted line breaks after {
formattedText = IndentLines(textBuffer, spanToFormat.Start, ast, formattedText, options, scopeStatementPosition, respectUserIndent);
if (!spanText.Equals(formattedText, StringComparison.Ordinal)) {
var selectionTracker = new RSelectionTracker(textView, textBuffer);
RTokenizer tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection<RToken> oldTokens = tokenizer.Tokenize(spanText);
IReadOnlyTextRangeCollection<RToken> newTokens = tokenizer.Tokenize(formattedText);
IncrementalTextChangeApplication.ApplyChangeByTokens(
textBuffer,
new TextStream(spanText), new TextStream(formattedText),
oldTokens, newTokens,
formatRange,
Resources.AutoFormat, selectionTracker);
return true;
}
return false;
}
public static string GetItemAtPosition(ITextSnapshotLine line, int position, Func <RTokenType, bool> tokenTypeCheck, out Span span)
{
string lineText = line.GetText();
var offset = 0;
var positionInTokens = position - line.Start;
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(lineText);
var tokenIndex = tokens.GetItemContaining(positionInTokens);
if (tokenIndex >= 0)
{
var token = tokens[tokenIndex];
if (token.TokenType == RTokenType.Comment)
{
// Tokenize inside comment since we do want F1 to work inside
// commented out code, code samples or Roxygen blocks.
positionInTokens -= token.Start;
var positionAfterHash = token.Start + 1;
tokens = tokenizer.Tokenize(lineText.Substring(positionAfterHash, token.Length - 1));
tokenIndex = tokens.GetItemContaining(positionInTokens);
if (tokenIndex >= 0)
{
token = tokens[tokenIndex];
offset = positionAfterHash;
}
}
if (tokenTypeCheck(token.TokenType))
{
var start = token.Start + offset;
var end = Math.Min(start + token.Length, line.End);
span = Span.FromBounds(line.Start + start, line.Start + end); // return view span
return(lineText.Substring(start, end - start));
}
}
span = Span.FromBounds(0, 0);
return(string.Empty);
}
public static void TokenizeFileImplementation(CoreTestFilesFixture fixture, string name)
{
string testFile = fixture.GetDestinationPath(name);
string baselineFile = testFile + ".tokens";
string text = fixture.LoadDestinationFile(name);
ITextProvider textProvider = new TextStream(text);
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(textProvider, 0, textProvider.Length);
string actual = DebugWriter.WriteTokens <RToken, RTokenType>(tokens);
if (_regenerateBaselineFiles)
{
// Update this to your actual enlistment if you need to update baseline
baselineFile = Path.Combine(fixture.SourcePath, @"Tokenization\", Path.GetFileName(testFile)) + ".tokens";
TestFiles.UpdateBaseline(baselineFile, actual);
}
else
{
TestFiles.CompareToBaseLine(baselineFile, actual);
}
}
/// <summary>
/// Should this key press trigger a completion session?
/// </summary>
public override bool IsTriggerChar(char typedCharacter)
{
if (!HasActiveCompletionSession)
{
switch (typedCharacter)
{
case '$':
case '@':
return(true);
case ':':
return(RCompletionContext.IsCaretInNamespace(TextView));
case '(':
return(RCompletionContext.IsCaretInLibraryStatement(TextView));
default:
if (_settings.ShowCompletionOnFirstChar)
{
SnapshotPoint?position = REditorDocument.MapCaretPositionFromView(TextView);
if (position.HasValue)
{
int pos = position.Value;
var snapshot = position.Value.Snapshot;
// Trigger on first character
if (RTokenizer.IsIdentifierCharacter(typedCharacter) && !char.IsDigit(typedCharacter))
{
// Ignore if this is not the first character
return(pos <= 1 || (pos > 1 && !RTokenizer.IsIdentifierCharacter(snapshot[pos - 2])));
}
}
}
break;
}
}
return(false);
}
private static CompletionList OrderList(List <ICompletionEntry> completions)
{
// Place 'name =' at the top prioritizing argument names
// Place items starting with non-alpha characters like .Call and &&
// at the end of the list.
var orderedCompletions = new List <Completion>();
var specialNames = new List <Completion>();
var generalEntries = new List <Completion>();
foreach (var c in completions)
{
if (RTokenizer.IsIdentifierCharacter(c.DisplayText[0]) && c.DisplayText.EndsWith("=", StringComparison.Ordinal))
{
// Place argument completions first
orderedCompletions.Add(new RCompletion(c));
}
else if (c.DisplayText.IndexOfIgnoreCase(".rtvs") < 0)
{
// Exclude .rtvs
if (!char.IsLetter(c.DisplayText[0]))
{
// Special names will come last
specialNames.Add(new RCompletion(c));
}
else
{
generalEntries.Add(new RCompletion(c));
}
}
}
orderedCompletions.AddRange(generalEntries);
orderedCompletions.AddRange(specialNames);
return(new CompletionList(orderedCompletions));
}
public override CommandResult Invoke(Guid group, int id, object inputArg, ref object outputArg) {
if (!TextView.Caret.InVirtualSpace) {
int caretPosition = TextView.Caret.Position.BufferPosition.Position;
SnapshotPoint? rPosition = TextView.MapDownToR(caretPosition);
if (rPosition.HasValue) {
int rCaretPosition = rPosition.Value.Position;
ITextSnapshotLine line = rPosition.Value.Snapshot.GetLineFromPosition(rCaretPosition);
// Tokenize current line
if (line != null) {
Span? spanToSelect = null;
var text = line.GetText();
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(text);
var positionInLine = rCaretPosition - line.Start;
var token = tokens.FirstOrDefault(t => t.Contains(positionInLine));
if (token != null) {
if (token.TokenType == RTokenType.String) {
// Select word inside string
spanToSelect = GetWordSpan(text, line.Start, positionInLine);
} else {
spanToSelect = new Span(token.Start + line.Start, token.Length);
}
}
if (spanToSelect.HasValue && spanToSelect.Value.Length > 0) {
NormalizedSnapshotSpanCollection spans = TextView.BufferGraph.MapUpToBuffer(
new SnapshotSpan(rPosition.Value.Snapshot, spanToSelect.Value),
SpanTrackingMode.EdgePositive, TextView.TextBuffer);
if (spans.Count == 1) {
TextView.Selection.Select(new SnapshotSpan(TextView.TextBuffer.CurrentSnapshot, spans[0]), isReversed: false);
return CommandResult.Executed;
}
}
}
}
}
return CommandResult.NotSupported;
}
public static bool FormatRangeExact(ITextView textView, ITextBuffer textBuffer, ITextRange formatRange, RFormatOptions options, IEditorShell editorShell) {
ITextSnapshot snapshot = textBuffer.CurrentSnapshot;
Span spanToFormat = new Span(formatRange.Start, formatRange.Length);
string spanText = snapshot.GetText(spanToFormat.Start, spanToFormat.Length);
string trimmedSpanText = spanText.Trim();
RFormatter formatter = new RFormatter(options);
string formattedText = formatter.Format(trimmedSpanText);
formattedText = formattedText.Trim(); // There may be inserted line breaks after {
// Apply formatted text without indentation. We then will update the parse tree
// so we can calculate proper line indents from the AST via the smart indenter.
if (!spanText.Equals(formattedText, StringComparison.Ordinal)) {
// Extract existing indent before applying changes. Existing indent
// may be used by the smart indenter for function argument lists.
var startLine = snapshot.GetLineFromPosition(spanToFormat.Start);
var originalIndentSizeInSpaces = IndentBuilder.TextIndentInSpaces(startLine.GetText(), options.IndentSize);
var selectionTracker = new RSelectionTracker(textView, textBuffer, formatRange);
RTokenizer tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection<RToken> oldTokens = tokenizer.Tokenize(spanText);
IReadOnlyTextRangeCollection<RToken> newTokens = tokenizer.Tokenize(formattedText);
IncrementalTextChangeApplication.ApplyChangeByTokens(
textBuffer,
new TextStream(spanText), new TextStream(formattedText),
oldTokens, newTokens,
formatRange,
Resources.AutoFormat, selectionTracker, editorShell,
() => {
var ast = UpdateAst(textBuffer);
// Apply indentation
IndentLines(textView, textBuffer, new TextRange(formatRange.Start, formattedText.Length), ast, options, originalIndentSizeInSpaces);
});
return true;
}
return false;
}
/// <summary>
/// Given position in the buffer tries to detemine start of the expression.
/// </summary>
private static int FindStartOfExpression(ITextBuffer textBuffer, int position) {
// Go up line by line, tokenize each line
// and check if it starts or ends with an operator
int lineNum = textBuffer.CurrentSnapshot.GetLineNumberFromPosition(position);
var tokenizer = new RTokenizer(separateComments: true);
var text = textBuffer.CurrentSnapshot.GetLineFromLineNumber(lineNum).GetText();
var tokens = tokenizer.Tokenize(text);
bool nextLineStartsWithOperator = tokens.Count > 0 && tokens[0].TokenType == RTokenType.Operator;
for (int i = lineNum - 1; i >= 0; i--) {
text = textBuffer.CurrentSnapshot.GetLineFromLineNumber(i).GetText();
tokens = tokenizer.Tokenize(text);
if (tokens.Count > 0) {
if (!nextLineStartsWithOperator && tokens[tokens.Count - 1].TokenType != RTokenType.Operator) {
break;
}
position = tokens[0].Start;
nextLineStartsWithOperator = tokens[0].TokenType == RTokenType.Operator;
}
}
return position;
}
public static IReadOnlyList<ISignatureInfo> ParseSignatures(string usageContent, IReadOnlyDictionary<string, string> argumentsDescriptions = null) {
// RD signature text may contain \dots sequence which denotes ellipsis.
// R parser does not know about it and hence we will replace \dots by ...
// Also, signatures may contain S3 method info like
// '\method{as.matrix}{data.frame}(x, rownames.force = NA, \dots)'
// which we need to filter out since they are irrelevant to intellisense.
List<ISignatureInfo> signatures = new List<ISignatureInfo>();
usageContent = usageContent.Replace(@"\dots", "...");
RTokenizer tokenizer = new RTokenizer(separateComments: true);
IReadOnlyTextRangeCollection<RToken> collection = tokenizer.Tokenize(usageContent);
ITextProvider textProvider = new TextStream(usageContent);
TokenStream<RToken> tokens = new TokenStream<RToken>(collection, RToken.EndOfStreamToken);
var parseContext = new ParseContext(textProvider,
TextRange.FromBounds(tokens.CurrentToken.Start, textProvider.Length),
tokens, tokenizer.CommentTokens);
while (!tokens.IsEndOfStream()) {
// Filter out '\method{...}{}(signature)
if (tokens.CurrentToken.TokenType == RTokenType.OpenCurlyBrace) {
// Check if { is preceded by \method
}
if (tokens.CurrentToken.TokenType != RTokenType.Identifier) {
break;
}
string functionName = textProvider.GetText(tokens.CurrentToken);
tokens.MoveToNextToken();
ISignatureInfo info = ParseSignature(functionName, parseContext, argumentsDescriptions);
if (info != null) {
signatures.Add(info);
}
}
return signatures;
}
/// <summary>
/// Provides list of completion entries for a given location in the AST.
/// </summary>
/// <param name="tree">Document tree</param>
/// <param name="position">Caret position in the document</param>
/// <param name="autoShownCompletion">True if completion is forced (like when typing Ctrl+Space)</param>
/// <returns>List of completion entries for a given location in the AST</returns>
public static IReadOnlyCollection <IRCompletionListProvider> GetCompletionForLocation(RCompletionContext context, bool autoShownCompletion)
{
List <IRCompletionListProvider> providers = new List <IRCompletionListProvider>();
if (context.AstRoot.Comments.Contains(context.Position))
{
// No completion in comments
return(providers);
}
// First check file completion - it happens inside strings
string directory;
if (CanShowFileCompletion(context.AstRoot, context.Position, out directory))
{
if (!string.IsNullOrEmpty(directory))
{
providers.Add(new FilesCompletionProvider(directory));
}
return(providers);
}
// Now check if position is inside a string and if so, suppress completion list
var tokenNode = context.AstRoot.GetNodeOfTypeFromPosition <TokenNode>(context.Position);
if (tokenNode != null && tokenNode.Token.TokenType == RTokenType.String)
{
// No completion in string
return(providers);
}
// Identifier character is a trigger but only as a first character so it doesn't suddenly
// bring completion back on a second character if user dismissed it after the first one,
// or in a middle of 'install.packages' when user types dot or in floating point numbers.
if (context.Position > 1 && autoShownCompletion)
{
char triggerChar = context.TextBuffer.CurrentSnapshot.GetText(context.Position - 1, 1)[0];
char charBeforeTigger = context.TextBuffer.CurrentSnapshot.GetText(context.Position - 2, 1)[0];
if (RTokenizer.IsIdentifierCharacter(triggerChar) && RTokenizer.IsIdentifierCharacter(charBeforeTigger))
{
return(providers);
}
}
if (IsInFunctionArgumentName <FunctionDefinition>(context.AstRoot, context.Position))
{
// No completion in function definition argument names
return(providers);
}
if (IsInObjectMemberName(context.AstRoot.TextProvider, context.Position))
{
providers.Add(new WorkspaceVariableCompletionProvider());
return(providers);
}
if (IsPackageListCompletion(context.TextBuffer, context.Position))
{
providers.Add(new PackagesCompletionProvider());
}
else
{
if (IsInFunctionArgumentName <FunctionCall>(context.AstRoot, context.Position))
{
providers.Add(new ParameterNameCompletionProvider());
}
foreach (var p in CompletionProviders)
{
providers.Add(p.Value);
}
if (!context.IsInNameSpace())
{
providers.Add(new PackagesCompletionProvider());
}
}
providers.Add(new WorkspaceVariableCompletionProvider());
return(providers);
}
private static bool IsStandaloneOperator(string text)
{
var tokens = new RTokenizer().Tokenize(text);
return(tokens.Count == 1 && tokens[0].TokenType == RTokenType.Operator);
}
public static string GetItemAtPosition(ITextSnapshotLine line, int position, Func<RTokenType, bool> tokenTypeCheck, out Span span) {
string lineText = line.GetText();
var offset = 0;
var positionInTokens = position - line.Start;
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(lineText);
var tokenIndex = tokens.GetItemContaining(positionInTokens);
if (tokenIndex >= 0) {
var token = tokens[tokenIndex];
if (token.TokenType == RTokenType.Comment) {
// Tokenize inside comment since we do want F1 to work inside
// commented out code, code samples or Roxygen blocks.
positionInTokens -= token.Start;
offset = token.Start + 1;
tokens = tokenizer.Tokenize(lineText.Substring(offset, token.Length - 1));
tokenIndex = tokens.GetItemContaining(positionInTokens);
if (tokenIndex >= 0) {
token = tokens[tokenIndex];
}
}
if (tokenTypeCheck(token.TokenType)) {
var start = token.Start + offset;
var end = Math.Min(start + token.Length, line.End);
span = Span.FromBounds(line.Start + start, line.Start + end); // return view span
return lineText.Substring(start, end - start);
}
}
span = Span.FromBounds(0, 0);
return string.Empty;
}
private static int PositionFromTokens(ITextSnapshot snapshot, int itemIndex, int offset) {
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(new TextProvider(snapshot), 0, snapshot.Length, true);
if (itemIndex >= 0 && itemIndex < tokens.Count) {
int position = tokens[itemIndex].Start - offset;
position = Math.Min(position, snapshot.Length);
position = Math.Max(position, 0);
return position;
}
return snapshot.Length;
}
private static bool IsMultiLineCandidate(string text) {
if (text.IndexOfAny(new[] { '\n', '\r' }) != -1) {
// if we already have newlines then we're multiline
return true;
}
var tokenizer = new RTokenizer();
IReadOnlyTextRangeCollection<RToken> tokens = tokenizer.Tokenize(new TextStream(text), 0, text.Length);
return tokens.Any(t => t.TokenType == RTokenType.OpenCurlyBrace);
}
/// <summary>
/// Tokenizes provided string that contains R code
/// </summary>
private void Tokenize(string text) {
_textProvider = new TextStream(text);
var tokenizer = new RTokenizer(separateComments: false);
var tokens = tokenizer.Tokenize(_textProvider, 0, _textProvider.Length);
_tokens = new TokenStream<RToken>(tokens, RToken.EndOfStreamToken);
_braceHandler = new BraceHandler(_tokens, _tb);
_expressionHelper = new ExpressionHelper(_tokens, _textProvider);
}
private static bool IsValidFunctionName(string functionName)
{
var tokens = new RTokenizer().Tokenize(functionName);
return(tokens.Count == 1 && tokens[0].TokenType == RTokenType.Identifier);
}
/// <summary>
/// Tokenizes provided string that contains R code
/// </summary>
private void Tokenize(string text) {
_textProvider = new TextStream(text);
var tokenizer = new RTokenizer(separateComments: false);
var tokens = tokenizer.Tokenize(_textProvider, 0, _textProvider.Length);
_tokens = new TokenStream<RToken>(tokens, RToken.EndOfStreamToken);
}
private int PositionFromTokens(ITextSnapshot snapshot, int itemIndex, int offset) {
int lengthChange = snapshot.Length - _lengthBeforeChange;
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(
new TextProvider(snapshot), _changingRange.Start, _changingRange.Length + lengthChange, true);
if (itemIndex >= 0 && itemIndex < tokens.Count) {
int position = tokens[itemIndex].Start - offset;
position = Math.Min(position, snapshot.Length);
position = Math.Max(position, 0);
return position;
}
return _changingRange.End + lengthChange;
}
/// <summary>
/// Handles R-like content in RD. This includes handling # and ##
/// as comments, counting brace nesting, handling "..." as string
/// (not true in plain RD LaTeX-like content) and colorizing numbers
/// by using actual R tokenizer. Now. there is a confusing part:
/// "There are two types of comments in R-like mode. As elsewhere in
/// Rd files, Rd comments start with %, and run to the end of the line."
/// If that is so then $ in sprintf will beging RD comment which frankly
/// doesn't make any sense fron the authoring/editing point of view.
/// "% characters must be escaped even within strings, or they will be
/// taken as Rd comments." Sure, but R engine doesn't do that when
/// requesting help in Rd format.
/// </summary>
private void HandleRContent()
{
var braceCounter = new BraceCounter <char>(new[] { '{', '}', '[', ']' });
while (!_cs.IsEndOfStream())
{
var handled = false;
switch (_cs.CurrentChar)
{
case '\"':
case '\'':
handled = HandleRString(_cs.CurrentChar);
break;
case '\\':
handled = IsEscape();
if (handled)
{
_cs.Advance(2);
}
else
{
handled = HandleKeyword();
}
break;
case '#':
handled = HandlePragma();
if (!handled)
{
if (_cs.NextChar == '#')
{
// ## is always comment in R-like content
handled = HandleComment();
}
else
{
// With a sinle # it may or may not be comment.
// For example, there are statements like \code{#}.
// Heuristic is to treat text that contains {} or \
// as NOT a comment.
var commentStart = _cs.Position;
_cs.SkipToEol();
var commentText = _cs.Text.GetText(TextRange.FromBounds(commentStart, _cs.Position));
_cs.Position = commentStart;
if (commentText.IndexOfAny(new[] { '{', '\\', '}' }) < 0)
{
handled = HandleComment();
}
}
}
break;
default:
if (braceCounter.CountBrace(_cs.CurrentChar))
{
handled = AddBraceToken();
if (braceCounter.Count == 0)
{
return;
}
}
else
{
// Check if sequence is a candidate for a number.
// The check is not perfect but numbers in R-like content
// are typically very simple as R blocks are usually
// code examples and do not contain exotic sequences.
if (!char.IsLetter(_cs.PrevChar) && (_cs.IsDecimal() || _cs.CurrentChar == '-' || _cs.CurrentChar == '.'))
{
var sequenceStart = _cs.Position;
_cs.SkipToWhitespace();
if (_cs.Position > sequenceStart)
{
var rt = new RTokenizer();
var candidate = _cs.Text.GetText(TextRange.FromBounds(sequenceStart, _cs.Position));
var rTokens = rt.Tokenize(candidate);
if (rTokens.Count > 0 && rTokens[0].TokenType == RTokenType.Number)
{
if (_tokenizeRContent)
{
AddToken(RdTokenType.Number, sequenceStart + rTokens[0].Start, rTokens[0].Length);
}
_cs.Position = sequenceStart + rTokens[0].End;
continue;
}
}
_cs.Position = sequenceStart;
}
}
break;
}
if (!handled)
{
_cs.MoveToNextChar();
}
}
}
public override bool Parse(ParseContext context, IAstNode parent)
{
RToken currentToken = context.Tokens.CurrentToken;
string text = context.TextProvider.GetText(currentToken);
double realPart = Double.NaN;
double imaginaryPart = Double.NaN;
Debug.Assert(currentToken.TokenType == RTokenType.Complex);
// Split into real and imaginary parts. Imaginary part
// should always be there since otherwise tokenizer would
// not have idenfified the number as complex. Note that
// real part may be missing as in '+0i'. Operator may also
// be missing: 1i is a legal complex number.
Debug.Assert(text[text.Length - 1] == 'i');
// Drop trailing i and retokenize as two numbers
RTokenizer tokenizer = new RTokenizer(separateComments: false);
IReadOnlyTextRangeCollection <RToken> tokens = tokenizer.Tokenize(text.Substring(0, text.Length - 1));
if (tokens.Count == 1)
{
// Only imaginary part is present
Debug.Assert(tokens[0].TokenType == RTokenType.Number);
if (!Number.TryParse(text.Substring(tokens[0].Start, tokens[0].Length), out imaginaryPart))
{
imaginaryPart = Double.NaN;
}
}
else if (tokens.Count == 3)
{
// Real and imaginary parts present
Debug.Assert(tokens[0].TokenType == RTokenType.Number);
Debug.Assert(tokens[1].TokenType == RTokenType.Operator);
Debug.Assert(tokens[2].TokenType == RTokenType.Number);
string real = text.Substring(tokens[0].Start, tokens[0].Length);
if (!Number.TryParse(real, out realPart))
{
realPart = Double.NaN;
}
// Imaginary does not allow 'L' suffix
string imaginary = text.Substring(tokens[2].Start, tokens[2].Length);
if (!Number.TryParse(imaginary, out imaginaryPart, allowLSuffix: false))
{
imaginaryPart = Double.NaN;
}
}
if (realPart == Double.NaN || imaginaryPart == Double.NaN)
{
context.AddError(new MissingItemParseError(ParseErrorType.NumberExpected, context.Tokens.PreviousToken));
return(false);
}
Complex complex = new Complex(realPart, imaginaryPart);
Value = new RComplex(complex);
return(base.Parse(context, parent));
}
/// <summary>
/// Extracts complete variable name under the caret. In '`abc`$`def` returns
/// the complete expression rather than its parts. Typically used to get data
/// for completion of variable members as in when user typed 'abc$def$'
/// Since method does not perform semantic analysis, it does not guaratee
/// syntactically correct expression, it may return 'a$$b'.
/// </summary>
public static string GetVariableNameBeforeCaret(this ITextView textView) {
if (textView.Caret.InVirtualSpace) {
return string.Empty;
}
var position = textView.Caret.Position.BufferPosition;
var line = position.GetContainingLine();
// For performance reasons we won't be using AST here
// since during completion it is most probably damaged.
string lineText = line.GetText();
var tokenizer = new RTokenizer();
var tokens = tokenizer.Tokenize(lineText);
var tokenPosition = position - line.Start;
var index = tokens.GetFirstItemBeforePosition(tokenPosition);
// Preceding token must be right next to caret position
if (index < 0 || tokens[index].End < tokenPosition || !IsVariableNameToken(lineText, tokens[index])) {
return string.Empty;
}
if (index == 0) {
return IsVariableNameToken(lineText, tokens[0]) ? lineText.Substring(tokens[0].Start, tokens[0].Length) : string.Empty;
}
// Walk back through tokens allowing identifier and specific
// operator tokens. No whitespace is permitted between tokens.
// We have at least 2 tokens here.
int i = index;
for (; i > 0; i--) {
var precedingToken = tokens[i - 1];
var currentToken = tokens[i];
if (precedingToken.End < currentToken.Start || !IsVariableNameToken(lineText, precedingToken)) {
break;
}
}
return lineText.Substring(tokens[i].Start, tokens[index].End - tokens[i].Start);
}
