public override void Add(string message, NewLineLocation[] lineLocations, int startIndex, int endIndex, int errorCode, Severity severity) {
if (severity == Severity.Error || severity == Severity.FatalError) {
_errors.Add(new ErrorResult(message, new SourceSpan(NewLineLocation.IndexToLocation(lineLocations, startIndex), NewLineLocation.IndexToLocation(lineLocations, endIndex))));
} else if (severity == Severity.Warning) {
_warnings.Add(new ErrorResult(message, new SourceSpan(NewLineLocation.IndexToLocation(lineLocations, startIndex), NewLineLocation.IndexToLocation(lineLocations, endIndex))));
}
}
public static IEnumerable <LineInfo> SplitLines(string text, int firstLineNumber = 0)
{
NewLineLocation nextLine;
var lineNo = firstLineNumber;
var lastLineEnd = 0;
while ((nextLine = NewLineLocation.FindNewLine(text, lastLineEnd)).EndIndex != lastLineEnd)
{
yield return(new LineInfo(
lastLineEnd,
nextLine.EndIndex - lastLineEnd - nextLine.Kind.GetSize(),
lineNo++,
nextLine.Kind
));
lastLineEnd = nextLine.EndIndex;
}
if (lastLineEnd != text.Length)
{
yield return(new LineInfo(
lastLineEnd,
text.Length - lastLineEnd,
lineNo++,
NewLineKind.None
));
}
}
internal static void AddChildNodeStepsOnNewLines <T>(
this List <ISourceCodeBuilderStep> steps,
IReadOnlyCollection <T> childNodes,
NewLineLocation location = NewLineLocation.AfterNode,
Func <SourceCodeStepsBuilder> builderFactory = null)
where T : IVisitable <IGenericVisitor>
{
if (builderFactory == null)
{
builderFactory = () => null;
}
foreach (T childNode in childNodes)
{
if (location == NewLineLocation.BeforeNode)
{
steps.Add(new WriteIndentedNewLine());
}
steps.Add(new WriteChildNode <T>(childNode, builderFactory()));
if (location == NewLineLocation.AfterNode)
{
steps.Add(new WriteIndentedNewLine());
}
}
}
public void Update(IEnumerable <DocumentChange> changes)
{
lock (_lock) {
_sb = _sb ?? new StringBuilder(_content);
foreach (var change in changes)
{
// Every change may change where the lines end so in order
// to correctly determine line/offsets we must re-split buffer
// into lines after each change.
var lineLoc = GetNewLineLocations().ToArray();
if (change.ReplaceAllText)
{
_sb = new StringBuilder(change.InsertedText);
continue;
}
var start = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.Start, _sb.Length);
var end = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.End, _sb.Length);
if (end > start)
{
_sb.Remove(start, end - start);
}
if (!string.IsNullOrEmpty(change.InsertedText))
{
_sb.Insert(start, change.InsertedText);
}
}
Version++;
_content = null;
}
}
public static IEnumerable <LineInfo> SplitLines(string text)
{
NewLineLocation nextLine;
List <LineInfo> lines = new List <LineInfo>();
int lastLineEnd = 0;
while ((nextLine = NewLineLocation.FindNewLine(text, lastLineEnd)).EndIndex != lastLineEnd)
{
yield return(new LineInfo(
lastLineEnd,
nextLine.EndIndex - lastLineEnd - nextLine.Kind.GetSize(),
lines.Count,
nextLine.Kind
));
lastLineEnd = nextLine.EndIndex;
}
if (lastLineEnd != text.Length)
{
yield return(new LineInfo(
lastLineEnd,
text.Length - lastLineEnd,
lines.Count,
NewLineKind.None
));
}
}
public SourceLocation Translate(SourceLocation loc, int fromVersion, int toVersion)
{
var snapVer = _snapshots.Peek().Version;
var fromVer = snapVer;
while (fromVer.Next != null && fromVer.VersionNumber < fromVersion)
{
fromVer = fromVer.Next;
}
var toVer = toVersion > fromVersion ? fromVer : snapVer;
while (toVer.Next != null && toVer.VersionNumber < toVersion)
{
toVer = toVer.Next;
}
var fromLines = GetLineLocations(fromVer.VersionNumber);
var index = NewLineLocation.LocationToIndex(fromLines, loc, fromVer.Length);
if (fromVer.VersionNumber < toVer.VersionNumber)
{
index = Tracking.TrackPositionForwardInTime(PointTrackingMode.Negative, index, fromVer, toVer);
}
else
{
index = Tracking.TrackPositionBackwardInTime(PointTrackingMode.Negative, index, fromVer, toVer);
}
var toLines = GetLineLocations(toVer.VersionNumber);
return(NewLineLocation.IndexToLocation(toLines, index));
}
public void Update(IEnumerable <DocumentChange> changes)
{
_sb = _sb ?? new StringBuilder(_content);
_content = null;
var lastStart = int.MaxValue;
var lineLoc = SplitLines(_sb).ToArray();
foreach (var change in changes)
{
var start = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.Start, _sb.Length);
if (start > lastStart)
{
throw new InvalidOperationException("changes must be in reverse order of start location");
}
lastStart = start;
var end = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.End, _sb.Length);
if (end > start)
{
_sb.Remove(start, end - start);
}
if (!string.IsNullOrEmpty(change.InsertedText))
{
_sb.Insert(start, change.InsertedText);
}
}
Version++;
}
public SourceSpan GetTokenSpan(IndexSpan span)
{
return(new SourceSpan(
NewLineLocation.IndexToLocation(TokenNewlines, span.Start),
NewLineLocation.IndexToLocation(TokenNewlines, span.End)
));
}
public void Update(DocumentChangeSet set)
{
Check.InvalidOperation(() => _ownerThreadId == Thread.CurrentThread.ManagedThreadId,
"Document buffer update must be done from the thread that created it");
if (!set.Changes.Any(c => c.WholeBuffer))
{
if (Version >= 0)
{
if (set.FromVersion < Version)
{
return;
}
if (set.FromVersion > Version)
{
throw new InvalidOperationException("missing prior versions");
}
}
if (set.FromVersion >= set.ToVersion)
{
throw new InvalidOperationException("cannot reduce version without resetting buffer");
}
}
var lastStart = int.MaxValue;
var lineLoc = SplitLines(_sb).ToArray();
foreach (var change in set.Changes)
{
if (change.WholeBuffer)
{
_sb.Clear();
if (!string.IsNullOrEmpty(change.InsertedText))
{
_sb.Append(change.InsertedText);
}
continue;
}
var start = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.Start, _sb.Length);
if (start > lastStart)
{
throw new InvalidOperationException("changes must be in reverse order of start location");
}
lastStart = start;
var end = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.End, _sb.Length);
if (end > start)
{
_sb.Remove(start, end - start);
}
if (!string.IsNullOrEmpty(change.InsertedText))
{
_sb.Insert(start, change.InsertedText);
}
}
Version = set.ToVersion;
}
private SourceSpan GetTokenSpan(IndexSpan span)
{
EnsureTokens();
return(new SourceSpan(
NewLineLocation.IndexToLocation(_tokenNewlines, span.Start),
NewLineLocation.IndexToLocation(_tokenNewlines, span.End)
));
}
public PythonAst(Statement body, NewLineLocation[] lineLocations, PythonLanguageVersion langVersion)
{
if (body == null) {
throw new ArgumentNullException("body");
}
_langVersion = langVersion;
_body = body;
_lineLocations = lineLocations;
}
internal void Add(string message, NewLineLocation[] lineLocations, int startIndex, int endIndex, int errorCode, Severity severity)
{
Add(
message,
new SourceSpan(NewLineLocation.IndexToLocation(lineLocations, startIndex), NewLineLocation.IndexToLocation(lineLocations, endIndex)),
errorCode,
severity
);
}
public void Update(DocumentChangeSet changes)
{
if (!changes.Changes.Any(c => c.WholeBuffer))
{
if (Version >= 0)
{
if (changes.FromVersion < Version)
{
return;
}
else if (changes.FromVersion > Version)
{
throw new InvalidOperationException("missing prior versions");
}
}
if (changes.FromVersion >= changes.ToVersion)
{
throw new InvalidOperationException("cannot reduce version without resetting buffer");
}
}
int lastStart = int.MaxValue;
var lineLoc = SplitLines(Text).ToArray();
foreach (var change in changes.Changes)
{
if (change.WholeBuffer)
{
Text.Clear();
if (!string.IsNullOrEmpty(change.InsertedText))
{
Text.Append(change.InsertedText);
}
continue;
}
int start = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.Start, Text.Length);
if (start > lastStart)
{
throw new InvalidOperationException("changes must be in reverse order of start location");
}
lastStart = start;
int end = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.End, Text.Length);
if (end > start)
{
Text.Remove(start, end - start);
}
if (!string.IsNullOrEmpty(change.InsertedText))
{
Text.Insert(start, change.InsertedText);
}
}
Version = changes.ToVersion;
}
public override void Add(string message, NewLineLocation[] lineLocations, int startIndex, int endIndex, int errorCode, Severity severity)
{
if (severity == Severity.Error || severity == Severity.FatalError)
{
_errors.Add(new ErrorResult(message, new SourceSpan(NewLineLocation.IndexToLocation(lineLocations, startIndex), NewLineLocation.IndexToLocation(lineLocations, endIndex))));
}
else if (severity == Severity.Warning)
{
_warnings.Add(new ErrorResult(message, new SourceSpan(NewLineLocation.IndexToLocation(lineLocations, startIndex), NewLineLocation.IndexToLocation(lineLocations, endIndex))));
}
}
private static void WriteEquals(ConsoleColor color, NewLineLocation lineLocation = NewLineLocation.None)
{
if (lineLocation.IsFlagSet(NewLineLocation.Before))
{
Console.WriteLine(Environment.NewLine);
}
WriteLineWithColor(color, new string('=', 125));
if (lineLocation.IsFlagSet(NewLineLocation.After))
{
Console.WriteLine(Environment.NewLine);
}
}
private static void AppendWithNewLines(StringBuilder result, NewLineLocation newLineLocation, string content)
{
if (newLineLocation.HasFlag(NewLineLocation.Before))
{
result.AppendLine();
}
result.Append(content);
if (newLineLocation.HasFlag(NewLineLocation.After))
{
result.AppendLine();
}
}
private static IEnumerable<NewLineLocation> SplitLines(StringBuilder text) {
NewLineLocation nextLine;
// TODO: Avoid string allocation by operating directly on StringBuilder
var str = text.ToString();
int lastLineEnd = 0;
while ((nextLine = NewLineLocation.FindNewLine(str, lastLineEnd)).EndIndex != lastLineEnd) {
yield return nextLine;
lastLineEnd = nextLine.EndIndex;
}
if (lastLineEnd != str.Length) {
yield return nextLine;
}
}
public void Update(IEnumerable <DocumentChange> changes)
{
lock (_lock) {
Check.InvalidOperation(_initialized, "Buffer is not initialized.");
if (_cleared)
{
return; // User may try and edit library file where we have already dropped the content.
}
_sb = _sb ?? new StringBuilder(_content);
foreach (var change in changes)
{
// Every change may change where the lines end so in order
// to correctly determine line/offsets we must re-split buffer
// into lines after each change.
var lineLoc = GetNewLineLocations().ToArray();
if (change.ReplaceAllText)
{
_sb = new StringBuilder(change.InsertedText);
continue;
}
var start = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.Start, _sb.Length);
var end = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.End, _sb.Length);
if (end > start)
{
_sb.Remove(start, end - start);
}
if (!string.IsNullOrEmpty(change.InsertedText))
{
_sb.Insert(start, change.InsertedText);
}
}
Version++;
_content = null;
}
}
public override void Add(string message, NewLineLocation[] lineLocations, int startIndex, int endIndex, int errorCode, Severity severity) {
_builder.AppendLine(message);
}
private static void AppendWithNewLines(StringBuilder result, NewLineLocation newLineLocation, string content)
{
if (newLineLocation.HasFlag(NewLineLocation.Before))
result.AppendLine();
result.Append(content);
if (newLineLocation.HasFlag(NewLineLocation.After))
result.AppendLine();
}
internal SourceLocation IndexToLocation(int index)
{
return(NewLineLocation.IndexToLocation(_lineLocations, index));
}
public int LocationToIndex(SourceLocation location)
{
return(NewLineLocation.LocationToIndex(_lineLocations, location, EndIndex));
}
public int GetIndex(SourceLocation loc, int atVersion)
{
var lines = GetLineLocations(atVersion);
return(NewLineLocation.LocationToIndex(lines, loc, -1));
}
public SourceLocation GetSourceLocation(int index, int atVersion)
{
var lines = GetLineLocations(atVersion);
return(NewLineLocation.IndexToLocation(lines, index));
}
public override void Add(string message, NewLineLocation[] lineLocations, int startIndex, int endIndex, int errorCode, Severity severity) {
if (_parser._errorCode == 0 && (severity == Severity.Error || severity == Severity.FatalError)) {
_parser._errorCode = errorCode;
}
_parser.ErrorSink.Add(message, lineLocations, startIndex, endIndex, errorCode, severity);
}
internal NewLineLocation[] GetLineLocations(int version)
{
var ver = _snapshots.Peek().Version;
NewLineLocation[] initial;
lock (_lineCache) {
// Precalculated for this version
if (_lineCache.TryGetValue(version, out initial))
{
return(initial);
}
// We simply can't provide these lines any more
if (ver.VersionNumber > version)
{
throw new NotSupportedException($"version {version} is no longer in memory");
}
int fromVersion = version;
// Get the last available set of newlines
while (--fromVersion >= ver.VersionNumber)
{
if (_lineCache.TryGetValue(fromVersion, out initial))
{
break;
}
}
// Create the initial set if it wasn't cached
if (initial == null)
{
fromVersion = ver.VersionNumber;
_lineCache[fromVersion] = initial = LinesToLineEnds(_snapshots.Peek().Lines).ToArray();
}
while (ver.Next != null && ver.VersionNumber < fromVersion)
{
ver = ver.Next;
}
#if DEBUG
var appliedVersions = new List <ITextVersion>();
#endif
List <NewLineLocation> asLengths = null;
while (ver.Changes != null && ver.VersionNumber < version)
{
#if DEBUG
appliedVersions.Add(ver);
#endif
if (asLengths == null)
{
asLengths = LineEndsToLineLengths(initial).ToList();
}
// Apply the changes from this version to the line lengths
foreach (var c in ver.Changes.Reverse())
{
var oldLoc = NewLineLocation.IndexToLocation(initial, c.OldPosition);
int lineNo = oldLoc.Line - 1;
while (asLengths.Count <= lineNo)
{
asLengths.Add(new NewLineLocation(0, NewLineKind.None));
}
if (c.OldLength > 0)
{
var line = asLengths[lineNo];
// Deletion may span lines, so combine them until we can delete
int cutAtCol = oldLoc.Column - 1;
for (int toRemove = c.OldLength; lineNo < asLengths.Count; lineNo += 1)
{
line = asLengths[lineNo];
int lineLen = line.EndIndex - cutAtCol;
cutAtCol = 0;
if (line.Kind == NewLineKind.CarriageReturnLineFeed && toRemove == lineLen - 1)
{
// Special case of deleting just the '\r' from a '\r\n' ending
asLengths[lineNo] = new NewLineLocation(line.EndIndex - toRemove, NewLineKind.LineFeed);
break;
}
else if (toRemove < lineLen)
{
asLengths[lineNo] = new NewLineLocation(line.EndIndex - toRemove, line.Kind);
break;
}
else
{
asLengths[lineNo] = new NewLineLocation(line.EndIndex - lineLen, NewLineKind.None);
toRemove -= lineLen;
if (toRemove <= 0)
{
break;
}
}
}
}
lineNo = oldLoc.Line - 1;
if (!string.IsNullOrEmpty(c.NewText))
{
var line = asLengths[lineNo];
NewLineLocation addedLine = new NewLineLocation(0, NewLineKind.None);
int lastLineEnd = 0, cutAtCol = oldLoc.Column - 1;
if (cutAtCol > line.EndIndex - line.Kind.GetSize() && lineNo + 1 < asLengths.Count)
{
cutAtCol = 0;
lineNo += 1;
line = asLengths[lineNo];
}
while ((addedLine = NewLineLocation.FindNewLine(c.NewText, lastLineEnd)).Kind != NewLineKind.None)
{
if (cutAtCol > 0)
{
asLengths[lineNo] = new NewLineLocation(line.EndIndex - cutAtCol, line.Kind);
lastLineEnd -= cutAtCol;
cutAtCol = 0;
}
line = new NewLineLocation(addedLine.EndIndex - lastLineEnd, addedLine.Kind);
asLengths.Insert(lineNo++, line);
lastLineEnd = addedLine.EndIndex;
}
if (addedLine.EndIndex > lastLineEnd)
{
if (lineNo < asLengths.Count)
{
line = asLengths[lineNo];
asLengths[lineNo] = line = new NewLineLocation(line.EndIndex + addedLine.EndIndex - lastLineEnd, line.Kind);
}
else
{
asLengths.Add(new NewLineLocation(addedLine.EndIndex - lastLineEnd, NewLineKind.None));
}
}
}
}
initial = LineLengthsToLineEnds(asLengths).ToArray();
if (asLengths.Count != initial.Length)
{
asLengths = null;
}
_lineCache[ver.VersionNumber + 1] = initial;
#if DEBUG
if (System.Diagnostics.Debugger.IsAttached && initial.Length > 0 && ver.Next != null && initial.Last().EndIndex != ver.Next.Length)
{
// Line calculations were wrong
// Asserts do not work properly here, so we just break if the debugger is attached
System.Diagnostics.Debugger.Break();
}
#endif
ver = ver.Next;
}
return(initial);
}
}
public override SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(_newLines, index);
public override int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(_newLines, location, _fileSize);
private static NewLineLocation[] GetEncodingLineNumbers(IList<byte> readBytes) {
NewLineLocation[] lineNos = new NewLineLocation[2];
for (int i = 0, lineCount = 0; i < readBytes.Count && lineCount < 2; i++) {
if (readBytes[i] == '\r') {
if (i + 1 < readBytes.Count && readBytes[i + 1] == '\n') {
lineNos[lineCount++] = new NewLineLocation(i, NewLineKind.CarriageReturnLineFeed);
i++;
} else {
lineNos[lineCount++] = new NewLineLocation(i, NewLineKind.CarriageReturn);
}
} else if (readBytes[i] == '\n') {
lineNos[lineCount++] = new NewLineLocation(i, NewLineKind.LineFeed);
}
}
return lineNos;
}
internal int LocationToIndex(SourceLocation location)
{
return(NewLineLocation.LocationToIndex(_lineLocations, location));
}
public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex);
public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index);
public virtual void Add(string message, NewLineLocation[] lineLocations, int startIndex, int endIndex, int errorCode, Severity severity) {
}