private static void ParseBlock(char[] buffer, int length, ILineBreaksEditor lineBreaks, ref LineEndingState lineEnding, ref int currentLineLength, ref int longestLineLength) { int index = 0; while (index < length) { int breakLength = TextUtilities.LengthOfLineBreak(buffer, index, length); if (breakLength == 0) { ++currentLineLength; ++index; } else { lineBreaks.Add(index, breakLength); longestLineLength = Math.Max(longestLineLength, currentLineLength); currentLineLength = 0; if (lineEnding != LineEndingState.Inconsistent) { if (breakLength == 2) { if (lineEnding == LineEndingState.Unknown) { lineEnding = LineEndingState.CRLF; } else if (lineEnding != LineEndingState.CRLF) { lineEnding = LineEndingState.Inconsistent; } } else { LineEndingState newLineEndingState; switch (buffer[index]) { // This code needs to be kep consistent with TextUtilities.LengthOfLineBreak() case '\r': newLineEndingState = LineEndingState.CR; break; case '\n': newLineEndingState = LineEndingState.LF; break; case '\u0085': newLineEndingState = LineEndingState.NEL; break; case '\u2028': newLineEndingState = LineEndingState.LS; break; case '\u2029': newLineEndingState = LineEndingState.PS; break; default: throw new InvalidOperationException("Unexpected line ending"); } if (lineEnding == LineEndingState.Unknown) { lineEnding = newLineEndingState; } else if (lineEnding != newLineEndingState) { lineEnding = LineEndingState.Inconsistent; } } } index += breakLength; } } }
internal static StringRebuilder Load(TextReader reader, long fileSize, string id, out bool hasConsistentLineEndings, out int longestLineLength, int blockSize = 0, int minCompressedBlockSize = TextImageLoader.BlockSize) // Exposed for unit tests { LineEndingState lineEnding = LineEndingState.Unknown; int currentLineLength = 0; longestLineLength = 0; bool useCompressedStringRebuilders = (fileSize >= TextModelOptions.CompressedStorageFileSizeThreshold); if (blockSize == 0) { blockSize = useCompressedStringRebuilders ? TextModelOptions.CompressedStoragePageSize : TextImageLoader.BlockSize; } PageManager pageManager = null; char[] buffer; if (useCompressedStringRebuilders) { pageManager = new PageManager(); buffer = new char[blockSize]; } else { buffer = TextImageLoader.AcquireBuffer(blockSize); } StringRebuilder content = StringRebuilderForChars.Empty; try { while (true) { int read = TextImageLoader.LoadNextBlock(reader, buffer); if (read == 0) { break; } var lineBreaks = LineBreakManager.CreateLineBreakEditor(read); TextImageLoader.ParseBlock(buffer, read, lineBreaks, ref lineEnding, ref currentLineLength, ref longestLineLength); char[] bufferForStringBuilder = buffer; if (read < (buffer.Length / 2)) { // We read far less characters than buffer so copy the contents to a new buffer and reuse the original buffer. bufferForStringBuilder = new char[read]; Array.Copy(buffer, bufferForStringBuilder, read); } else { // We're using most of bufferForStringRebuilder so allocate a new block for the next chunk. buffer = new char[blockSize]; } var newContent = (useCompressedStringRebuilders && (read > minCompressedBlockSize)) ? StringRebuilderForCompressedChars.Create(new Page(pageManager, bufferForStringBuilder, read), lineBreaks) : StringRebuilderForChars.Create(bufferForStringBuilder, read, lineBreaks); content = content.Insert(content.Length, newContent); } longestLineLength = Math.Max(longestLineLength, currentLineLength); hasConsistentLineEndings = lineEnding != LineEndingState.Inconsistent; } finally { if (!useCompressedStringRebuilders) { TextImageLoader.ReleaseBuffer(buffer); } } return(content); }
private static ILineBreaks ParseBlock(char[] buffer, int length, ref LineEndingState lineEnding, ref int currentLineLength, ref int longestLineLength) { // Note that the lineBreaks created here will (internally) use the pooled list of line breaks. IPooledLineBreaksEditor lineBreaks = LineBreakManager.CreatePooledLineBreakEditor(length); int index = 0; while (index < length) { int breakLength = TextUtilities.LengthOfLineBreak(buffer, index, length); if (breakLength == 0) { ++currentLineLength; ++index; } else { lineBreaks.Add(index, breakLength); longestLineLength = Math.Max(longestLineLength, currentLineLength); currentLineLength = 0; if (lineEnding != LineEndingState.Inconsistent) { if (breakLength == 2) { if (lineEnding == LineEndingState.Unknown) { lineEnding = LineEndingState.CRLF; } else if (lineEnding != LineEndingState.CRLF) { lineEnding = LineEndingState.Inconsistent; } } else { LineEndingState newLineEndingState; switch (buffer[index]) { // This code needs to be kep consistent with TextUtilities.LengthOfLineBreak() case '\r': newLineEndingState = LineEndingState.CR; break; case '\n': newLineEndingState = LineEndingState.LF; break; case '\u0085': newLineEndingState = LineEndingState.NEL; break; case '\u2028': newLineEndingState = LineEndingState.LS; break; case '\u2029': newLineEndingState = LineEndingState.PS; break; default: throw new InvalidOperationException("Unexpected line ending"); } if (lineEnding == LineEndingState.Unknown) { lineEnding = newLineEndingState; } else if (lineEnding != newLineEndingState) { lineEnding = LineEndingState.Inconsistent; } } } index += breakLength; } } lineBreaks.ReleasePooledLineBreaks(); return(lineBreaks); }