} // ReadOneByte
private char ReadOneChar(TextReader reader)
{
// NOTE: the handling of multi-byte encodings is probably not the most
// efficient here ...
var completed = false;
var byteIndex = 0;
while (!completed)
{
_byteDecodingBuffer[byteIndex] = (byte)ReadOneByte(reader);
byteIndex++;
int usedBytes;
int usedChars;
_byteToCharDecoder.Convert(
_byteDecodingBuffer, 0, byteIndex,
_charDecodingBuffer, 0, 1,
true,
out usedBytes,
out usedChars,
out completed);
if (completed && (usedBytes != byteIndex || usedChars != 1))
{
throw new RtfMultiByteEncodingException(Strings.InvalidMultiByteEncoding(
_byteDecodingBuffer, byteIndex, _encoding));
}
}
var character = _charDecodingBuffer[0];
return(character);
} // ReadOneChar
public static unsafe void ConvertNegativeTest()
{
Decoder decoder = Encoding.UTF8.GetDecoder();
int bytesUsed;
int charsUsed;
bool completed;
byte [] bytes = Encoding.UTF8.GetBytes("\u0D800\uDC00");
fixed(byte *bytesPtr = bytes)
fixed(char *charsPtr = new char[1])
{
byte *pBytes = bytesPtr;
char *pChars = charsPtr;
AssertExtensions.Throws <ArgumentNullException>("chars", () => decoder.Convert(pBytes, bytes.Length, null, 1, true, out bytesUsed, out charsUsed, out completed));
AssertExtensions.Throws <ArgumentNullException>("bytes", () => decoder.Convert(null, bytes.Length, pChars, 1, true, out bytesUsed, out charsUsed, out completed));
AssertExtensions.Throws <ArgumentOutOfRangeException>("byteCount", () => decoder.Convert(pBytes, -1, pChars, 1, true, out bytesUsed, out charsUsed, out completed));
AssertExtensions.Throws <ArgumentOutOfRangeException>("charCount", () => decoder.Convert(pBytes, bytes.Length, pChars, -1, true, out bytesUsed, out charsUsed, out completed));
AssertExtensions.Throws <ArgumentException>("chars", () => decoder.Convert(pBytes, bytes.Length, pChars, 0, true, out bytesUsed, out charsUsed, out completed));
}
decoder = Encoding.GetEncoding("us-ascii", new EncoderExceptionFallback(), new DecoderExceptionFallback()).GetDecoder();
fixed(byte *bytesPtr = new byte [] { 0xFF, 0xFF })
fixed(char *charsPtr = new char[2])
{
byte *pBytes = bytesPtr;
char *pChars = charsPtr;
Assert.Throws <DecoderFallbackException>(() => decoder.Convert(pBytes, 2, pChars, 2, true, out bytesUsed, out charsUsed, out completed));
}
}
private void DecodeBuffer(int bytesInBuffer)
{
_DecodedCharacterOffset = 0;
_DecodedCharacterCount = 0;
_InputBufferOffset = 0;
_InputBufferCount = bytesInBuffer;
int characterWriteOffset = 0, characterWriteCapacity = _DecodedBuffer.Length;
bool completed = false;
while (!completed)
{
int bytesUsed, charsUsed;
_Decoder.Convert(
_InputBuffer, _InputBufferOffset, _InputBufferCount,
_DecodedBuffer, characterWriteOffset, characterWriteCapacity,
false, out bytesUsed, out charsUsed, out completed
);
_InputBufferOffset += bytesUsed;
_InputBufferCount -= bytesUsed;
characterWriteOffset += charsUsed;
characterWriteCapacity -= charsUsed;
_DecodedCharacterCount += charsUsed;
}
}
internal String ReadString()
{
var builder = new StringBuilder(8); // most sort responses will be OK, QUEUED, etc..
var crfound = false;
var bytesUsed = 0;
var charUsed = 0;
var completed = false;
while (CanRead())
{
var start = _left;
for (; _left < _right; _left++)
{
var c = _buffer[_left];
if (c == LFByte && crfound)
{
_left++;
_decoder.Convert(_buffer, start, _left - start, _charBuffer, 0, _charBuffer.Length, true, out bytesUsed, out charUsed, out completed);
builder.Append(_charBuffer, 0, charUsed);
return(builder.ToString(0, builder.Length - 2));
}
crfound = c == CRByte;
}
_decoder.Convert(_buffer, start, _right - start, _charBuffer, 0, _charBuffer.Length, false, out bytesUsed, out charUsed, out completed);
builder.Append(_charBuffer, 0, charUsed);
}
throw new RedisClientSocketException("Cannot read a string, not available data.");
}
/// <summary>
/// Reads a single character from the byte buffer provided, putting it into the internal cache
/// </summary>
/// <param name="sbuffer">The buffer to read.</param>
/// <returns><c>true</c> if a character was found; <c>false</c> otherwise.</returns>
/// <exception cref="System.ArgumentNullException">buffer may not be null.</exception>
/// <remarks>
/// It is expected that this method be used iteratively to find a single character. Data is not consumed
/// from the <paramref name="sbuffer"/>, but state is updated in this class to advance the byte offset
/// into the buffer to the next character. Should data be consumed from the internal buffer, you will
/// need to reset the state of this object.
/// </remarks>
private bool PeekChar(SerialBuffer sbuffer)
{
// Once the bug from Mono is fixed, we just drop the code above.
if (sbuffer == null)
{
throw new ArgumentNullException("sbuffer");
}
int readLen = sbuffer.Serial.ReadBuffer.Length;
if (Log.ReadToTrace(System.Diagnostics.TraceEventType.Verbose))
{
Log.ReadTo.TraceEvent(System.Diagnostics.TraceEventType.Verbose, 0,
"PeekChar: readlen={0}; m_ReadOffset={1}; m_ReadCache.Free={2}", readLen, m_ReadOffset, m_ReadCache.Free);
}
if (m_ReadOffset >= readLen)
{
return(false);
}
if (m_ReadCache.Free <= 1)
{
Overflow();
}
char[] oneChar = new char[2];
int cu = 0;
while (cu == 0 && m_ReadOffset < readLen)
{
int bu; bool complete;
try {
// Some UTF8 sequences may result in two UTF16 characters being generated.
Decoder.Convert(sbuffer.Serial.ReadBuffer.Array,
sbuffer.Serial.ReadBuffer.ToArrayIndex(m_ReadOffset),
1, oneChar, 0, 1, false, out bu, out cu, out complete);
} catch (System.ArgumentException ex) {
if (!ex.ParamName.Equals("chars"))
{
throw;
}
Decoder.Convert(sbuffer.Serial.ReadBuffer.Array,
sbuffer.Serial.ReadBuffer.ToArrayIndex(m_ReadOffset),
1, oneChar, 0, 2, false, out bu, out cu, out complete);
}
m_ReadOffset += bu;
}
if (cu == 0)
{
return(false);
}
m_ReadCache.Append(oneChar, 0, cu);
m_ReadOffsets.Append(m_ReadOffset - m_LastChar);
if (cu > 1)
{
m_ReadOffsets.Append(0);
}
m_LastChar = m_ReadOffset;
return(true);
}
// Token: 0x060006EE RID: 1774 RVA: 0x0001B29C File Offset: 0x0001949C
public string ReadString(int length, Encoding encoding)
{
int num = length;
char[] array = new char[encoding.GetMaxCharCount(length)];
Decoder decoder = encoding.GetDecoder();
int i = this.m_CurrentSegmentIndex;
int num2 = this.m_CurrentSegmentOffset;
int num3 = 0;
while (i < this.m_Segments.Count)
{
ArraySegment <byte> arraySegment = this.m_Segments[i];
if (i != this.m_CurrentSegmentIndex)
{
num2 = arraySegment.Offset;
}
int num4;
if (num2 + length < arraySegment.Offset + arraySegment.Count)
{
int byteCount = length;
int charCount = array.Length - num3;
int num5;
bool flag;
decoder.Convert(arraySegment.Array, num2, byteCount, array, num3, charCount, true, out num4, out num5, out flag);
num3 += num5;
}
else
{
int num6 = arraySegment.Count - (num2 - arraySegment.Offset);
bool flush = num6 == length;
int charCount2 = array.Length - num3;
int num5;
bool flag;
decoder.Convert(arraySegment.Array, num2, num6, array, num3, charCount2, flush, out num4, out num5, out flag);
num3 += num5;
}
length -= num4;
num2 += num4;
if (length == 0)
{
break;
}
if (i != this.m_Segments.Count - 1)
{
i++;
}
}
this.m_CurrentSegmentIndex = i;
this.m_CurrentSegmentOffset = num2;
this.m_Position += (long)num;
return(new string(array, 0, num3));
}
/// <summary>
/// Reads a single character from the byte buffer provided, putting it into the internal cache
/// </summary>
/// <param name="buffer">The buffer to read.</param>
/// <returns><c>true</c> if a character was found; <c>false</c> otherwise.</returns>
/// <exception cref="System.ArgumentNullException">buffer may not be null.</exception>
/// <remarks>
/// It is expected that this method be used iteratively to find a single character.
/// </remarks>
public bool PeekChar(CircularBuffer <byte> buffer)
{
int bytesRead = 0;
if (buffer == null)
{
throw new ArgumentNullException("buffer");
}
int readlen = buffer.Length;
if (m_ReadOffset >= readlen)
{
return(false);
}
char[] oneChar = new char[2];
int cu = 0;
while (cu == 0 && m_ReadOffset < readlen)
{
int bu; bool complete;
try {
// Some UTF8 sequences may result in two UTF16 characters being generated.
Decoder.Convert(buffer.Array, buffer.ToArrayIndex(m_ReadOffset), 1, oneChar, 0, 1, false, out bu, out cu, out complete);
} catch (System.ArgumentException ex) {
if (!ex.ParamName.Equals("chars"))
{
throw;
}
Decoder.Convert(buffer.Array, buffer.ToArrayIndex(m_ReadOffset), 1, oneChar, 0, 2, false, out bu, out cu, out complete);
}
bytesRead += bu;
m_ReadOffset++;
}
if (cu == 0)
{
return(false);
}
if (m_ReadCache.Free <= 1)
{
Overflow();
}
m_ReadCache.Append(oneChar, 0, cu);
m_ReadOffsets.Append(m_ReadOffset - m_LastChar);
if (cu > 1)
{
m_ReadOffsets.Append(0);
}
m_LastChar = m_ReadOffset;
return(true);
}
public string ReadStringShort()
{
int len = stream.ReadByte();
stream.Read(byteBuf, 0, len);
IncreaseReadBytes(len + 1);
int charsUsed, bytesUsed; bool completed;
dec.Convert(byteBuf, 0, len, charBuf, 0, byte.MaxValue, true, out bytesUsed, out charsUsed, out completed);
return(new string(charBuf, 0, charsUsed));
}
public unsafe string ReadString()
{
int charCount = ReadInt32();
if (charCount == -1)
{
return(null);
}
string result = new string('\0', charCount);
fixed(char *chars = result)
{
Decoder decoder = Encoding.UTF8.GetDecoder();
int bytesUsed;
int charsUsed;
bool completed;
#if NETSTANDARD1_3
byte[] bytes = new byte[(int)Math.Min(int.MaxValue - 50, _capacity - _offset)];
Marshal.Copy(_viewAddress, bytes, 0, bytes.Length);
char[] charArray = new char[charCount];
decoder.Convert(bytes, 0, bytes.Length, charArray, 0, charArray.Length, false, out bytesUsed, out charsUsed, out completed);
Marshal.Copy(charArray, 0, (IntPtr)chars, charsUsed * sizeof(char));
#else
decoder.Convert((byte *)_viewAddress, (int)Math.Min(int.MaxValue - 50, _capacity - _offset), chars, charCount, false, out bytesUsed, out charsUsed, out completed);
#endif
_offset += bytesUsed;
if (!completed)
{
long availableInFile = _fileLength - _viewOffset - _offset;
Resize(checked ((int)Math.Min(availableInFile, Encoding.UTF8.GetMaxByteCount(charCount - charsUsed))));
int finalBytesUsed;
int finalCharsUsed;
#if NETSTANDARD1_3
bytes = new byte[(int)Math.Min(int.MaxValue - 50, _capacity - _offset)];
Marshal.Copy(_viewAddress + bytesUsed, bytes, 0, bytes.Length);
charArray = new char[charCount - charsUsed];
decoder.Convert(bytes, 0, bytes.Length, charArray, 0, charArray.Length, true, out finalBytesUsed, out finalCharsUsed, out completed);
Marshal.Copy(charArray, 0, (IntPtr)(chars + charsUsed), finalCharsUsed * sizeof(char));
#else
decoder.Convert((byte *)_viewAddress + bytesUsed, (int)Math.Min(int.MaxValue - 50, _capacity - _offset), chars + charsUsed, charCount - charsUsed, true, out finalBytesUsed, out finalCharsUsed, out completed);
#endif
}
}
return(result);
}
public string ReadString(int length, Encoding encoding)
{
int num = length;
char[] array = new char[encoding.GetMaxCharCount(length)];
Decoder decoder = encoding.GetDecoder();
int num2 = m_CurrentSegmentIndex;
int num3 = m_CurrentSegmentOffset;
int num4 = 0;
while (num2 < m_Segments.Count)
{
ArraySegment <byte> arraySegment = m_Segments[num2];
if (num2 != m_CurrentSegmentIndex)
{
num3 = arraySegment.Offset;
}
int bytesUsed;
int charsUsed;
bool completed;
if (num3 + length < arraySegment.Offset + arraySegment.Count)
{
int byteCount = length;
int charCount = array.Length - num4;
decoder.Convert(arraySegment.Array, num3, byteCount, array, num4, charCount, flush: true, out bytesUsed, out charsUsed, out completed);
num4 += charsUsed;
}
else
{
int num5 = arraySegment.Count - (num3 - arraySegment.Offset);
bool flush = num5 == length;
int charCount2 = array.Length - num4;
decoder.Convert(arraySegment.Array, num3, num5, array, num4, charCount2, flush, out bytesUsed, out charsUsed, out completed);
num4 += charsUsed;
}
length -= bytesUsed;
num3 += bytesUsed;
if (length == 0)
{
break;
}
if (num2 != m_Segments.Count - 1)
{
num2++;
}
}
m_CurrentSegmentIndex = num2;
m_CurrentSegmentOffset = num3;
m_Position += num;
return(new string(array, 0, num4));
}
public override char ReadChar()
{
char result;
fixed(byte *b = buf)
{
decoder.Convert(
b + pos, buf.Length - pos, &result, 1, true,
out int bytesUsed, out int charsUsed, out bool completed);
pos += bytesUsed;
}
return(result);
}
public bool IsWebRequestError(out string err)
{
err = string.Empty;
if (this.m_protocol == Protocol.FILE || this.useLoadFromCacheOrDownload)
{
return(false);
}
bool flag = this.m_protocol == Protocol.HTTP && !WWWItem.PageNotFoundParsing.isIgnore && this.m_www.size != 0 && this.m_www.size < 10000;
if (!flag)
{
return(false);
}
if (TsPlatform.IsIPhone && !this.assetPath.Contains(".jpg"))
{
flag = false;
}
if (flag)
{
char[] array = new char[20];
byte[] bytes = this.m_www.bytes;
Decoder decoder = Encoding.UTF8.GetDecoder();
int num;
int num2;
bool flag2;
decoder.Convert(bytes, 0, 20, array, 0, array.Length, true, out num, out num2, out flag2);
string text = new string(array);
if (text.Contains("!DOC"))
{
err = string.Format("404 Page not found! (url=\"{0}\")", this.m_www.url);
}
}
return(!string.IsNullOrEmpty(err));
}
public static bool DecodeString(this Decoder source, byte[] bytes, int bytesOffset, int bytesLength, char[] buffer, StringBuilder result)
{
bool isCompleted;
// Loop
do
{
int bytesUsed;
int charsUsed;
source.Convert(
bytes,
bytesOffset,
bytesLength,
buffer,
0,
buffer.Length,
false,
out bytesUsed,
out charsUsed,
out isCompleted
);
result.Append(buffer, 0, charsUsed);
bytesOffset += bytesUsed;
bytesLength -= bytesUsed;
} while (bytesLength > 0);
return(isCompleted);
}
private void Flush()
{
decoder.Convert(byteBuffer, 0, bbIndex, charBuffer, 0, 4, false, out int bytesUsed, out int charsUsed, out bool completed);
sb.Append(charBuffer, 0, charsUsed);
ResetBuffers();
}
internal protected override CoderResult decodeLoop(ByteBuffer @in, CharBuffer @out)
{
int byte_count = @in.remaining();
int char_count = @out.capacity();
if (byte_count < 1)
{
return(CoderResult.UNDERFLOW);
}
byte[] bytes = new byte [byte_count];
@in.get(bytes, 0, byte_count);
char[] chars = new char [char_count];
int bytes_used, chars_used;
bool completed;
decoder.Convert(
bytes, 0, byte_count, chars, 0, char_count, false,
out bytes_used, out chars_used, out completed);
if (bytes_used != byte_count)
{
return(CoderResult.OVERFLOW);
}
@out.put(chars, 0, chars_used);
return(CoderResult.UNDERFLOW);
}
private async Task WriteAsync(
ArraySegment <byte> bufferSegment,
bool flush,
CancellationToken cancellationToken)
{
var decoderCompleted = false;
while (!decoderCompleted)
{
_decoder.Convert(
bufferSegment,
_charBuffer.AsSpan(_charsDecoded),
flush,
out var bytesDecoded,
out var charsDecoded,
out decoderCompleted);
_charsDecoded += charsDecoded;
bufferSegment = bufferSegment.Slice(bytesDecoded);
if (flush || !decoderCompleted)
{
// This is being invoked from FlushAsync or the char buffer is not large enough
// to accomodate all writes.
await WriteBufferAsync(flush, cancellationToken);
}
}
}
private static void AppendPageData(StringBuilder page
, byte[] byteData
, int bytesRead
, int totalBytes
)
{
// Convert from bytes to chars, and add to the page.
int byteUsed;
int charUsed = 0;
bool completed = false;
var charData = new char[4096];
while (!completed)
{
UTF8decoder.Convert(byteData
, 0
, bytesRead
, charData
, 0
, bytesRead
, true
, out byteUsed
, out charUsed
, out completed
);
}
page.Append(new string( charData, 0, charUsed ));
// Display the page download status.
Debug.Print("Bytes Read Now: " + bytesRead + " Total: " + totalBytes);
}
private async Task WriteAsync(
ArraySegment <byte> bufferSegment,
CancellationToken cancellationToken)
{
var decoderCompleted = false;
while (!decoderCompleted)
{
_decoder.Convert(
bufferSegment,
_charBuffer.AsSpan(_charsDecoded),
flush: false,
out var bytesDecoded,
out var charsDecoded,
out decoderCompleted);
_charsDecoded += charsDecoded;
bufferSegment = bufferSegment.Slice(bytesDecoded);
if (!decoderCompleted)
{
await WriteBufferAsync(cancellationToken);
}
}
}
public static string ReadStream(string source, Stream Reader)
{
Reader.ReadTimeout = 1000;
byte[] byteData = new byte[4096];
char[] charData = new char[4096];
string data = null;
int bytesRead = 0;
int totalBytes = 0;
try
{
while ((bytesRead = Reader.Read(byteData, 0, byteData.Length)) > 0)
{
int byteUsed, charUsed;
bool completed = false;
totalBytes += bytesRead;
UTF8decoder.Convert(byteData, 0, bytesRead, charData, 0, bytesRead, true, out byteUsed, out charUsed, out completed);
data = data + new String(charData, 0, charUsed);
Log.Comment("[" + source + "] Bytes Read Now: " + bytesRead + " Total: " + totalBytes);
}
}
catch (System.Net.Sockets.SocketException se)
{
if (se.ErrorCode != (int)System.Net.Sockets.SocketError.TimedOut)
{
throw se;
}
}
Log.Comment("[" + source + "] Total bytes in message body : " + totalBytes);
Log.Comment("[" + source + "] Received: " + data);
return(data);
}
public int GetPositionForLine(int line)
{
Debug.Assert(line >= currentLine);
while (line > currentLine)
{
int b = fs.ReadByte();
if (b < 0)
{
throw new EndOfStreamException();
}
int bytesUsed, charsUsed;
bool completed;
input[0] = (byte)b;
decoder.Convert(input, 0, 1, output, 0, 1, false, out bytesUsed, out charsUsed, out completed);
Debug.Assert(bytesUsed == 1);
if (charsUsed == 1)
{
if ((prevChar != '\r' && output[0] == '\n') || output[0] == '\r')
{
currentLine++;
}
prevChar = output[0];
}
}
return(checked ((int)fs.Position));
}
/// <summary>
/// Decodes the given bytes into characters, and stores the leftover bytes for later use
/// </summary>
/// <param name="inBuffer">Buffer of bytes to decode</param>
/// <param name="inBufferCount">Number of bytes to decode from the inBuffer</param>
/// <param name="outBuffer">Buffer to write the characters to</param>
/// <param name="outBufferOffset">Offset to start writing to outBuffer at</param>
/// <param name="outBufferCount">Maximum number of characters to decode</param>
/// <returns>The actual number of characters decoded</returns>
private int DecodeBytesToChars(byte[] inBuffer, int inBufferCount, char[] outBuffer, int outBufferOffset, int outBufferCount)
{
Debug.Assert(inBuffer != null, "Null input buffer");
Debug.Assert((inBufferCount > 0) && (inBufferCount <= inBuffer.Length), $"Bad inBufferCount: {inBufferCount}");
Debug.Assert(outBuffer != null, "Null output buffer");
Debug.Assert((outBufferOffset >= 0) && (outBufferCount > 0) && (outBufferOffset + outBufferCount <= outBuffer.Length), $"Bad outBufferCount: {outBufferCount} or outBufferOffset: {outBufferOffset}");
int charsRead;
int bytesUsed;
bool completed;
_decoder.Convert(inBuffer, 0, inBufferCount, outBuffer, outBufferOffset, outBufferCount, false, out bytesUsed, out charsRead, out completed);
// completed may be false and there is no spare bytes if the Decoder has stored bytes to use later
if ((!completed) && (bytesUsed < inBufferCount))
{
_leftOverBytes = new byte[inBufferCount - bytesUsed];
Array.Copy(inBuffer, bytesUsed, _leftOverBytes, 0, _leftOverBytes.Length);
}
else
{
// If Convert() sets completed to true, then it must have used all of the bytes we gave it
Debug.Assert(bytesUsed >= inBufferCount, "Converted completed, but not all bytes were used");
_leftOverBytes = null;
}
Debug.Assert(((_reader == null) || (_reader.ColumnDataBytesRemaining() > 0) || (!completed) || (_leftOverBytes == null)), "Stream has run out of data and the decoder finished, but there are leftover bytes");
Debug.Assert(charsRead > 0, "Converted no chars. Bad encoding?");
return(charsRead);
}
public int GetPositionForLine(int line)
{
Debug.Assert(line >= currentLine);
var inputLocal = input;
var outputLocal = output;
while (line > currentLine)
{
int b = fs.ReadByte();
if (b < 0)
{
throw new EndOfStreamException();
}
int bytesUsed, charsUsed;
bool completed;
inputLocal[0] = (byte)b;
decoder.Convert(inputLocal, 0, 1, outputLocal, 0, outputLocal.Length, false, out bytesUsed, out charsUsed, out completed);
Debug.Assert(bytesUsed == 1);
for (int i = 0; i < charsUsed; i++)
{
if (outputLocal[i] == '\n')
{
currentLine++;
}
}
}
return(checked ((int)fs.Position));
}
private async ValueTask <bool> ReadInputChars(CancellationToken cancellationToken)
{
// If we had left-over bytes from a previous read, move it to the start of the buffer and read content in to
// the segment that follows.
Buffer.BlockCopy(
_byteBuffer.Array,
_byteBuffer.Offset,
_byteBuffer.Array,
0,
_byteBuffer.Count);
var readBytes = await _stream.ReadAsync(_byteBuffer.Array.AsMemory(_byteBuffer.Count), cancellationToken);
_byteBuffer = new ArraySegment <byte>(_byteBuffer.Array, 0, _byteBuffer.Count + readBytes);
Debug.Assert(_charBuffer.Count == 0, "We should only expect to read more input chars once all buffered content is read");
_decoder.Convert(
_byteBuffer.AsSpan(),
_charBuffer.Array,
flush: readBytes == 0,
out var bytesUsed,
out var charsUsed,
out _);
_byteBuffer = _byteBuffer.Slice(bytesUsed);
_charBuffer = new ArraySegment <char>(_charBuffer.Array, 0, charsUsed);
return(readBytes == 0);
}
internal static string PtrToStringUTF8(IntPtr ptr, int size)
{
if (ptr != IntPtr.Zero && size > 0)
{
var barray = new byte[size];
// copy utf8 encoded string to byte array
Marshal.Copy(ptr, barray, 0, size);
// convert the UTF8 encoded string to unicode
var utf8_encoding = new UTF8Encoding();
int newsize = utf8_encoding.GetCharCount(barray);
var darray = new char[newsize];
Decoder decoder = utf8_encoding.GetDecoder();
int bytes_used, chars_used;
bool completed;
decoder.Convert(
barray,
0,
barray.Length,
darray,
0,
newsize,
true,
out bytes_used,
out chars_used,
out completed);
return(new string(darray));
}
return(null);
}
unsafe static string FromUTF8(byte *src)
{
Decoder d = Encoding.UTF8.GetDecoder();
int nSrc = 0;
for (; nSrc < 1024 * 1024; nSrc++)
{
if (src[nSrc] == 0)
{
break;
}
}
nSrc++; //count = lastindex + 1
int nDest = d.GetCharCount(src, nSrc, true);
char[] chBuf = new char[nDest];
string res;
fixed(char *chOut = chBuf)
{
int bytesUsed;
int charsUsed;
bool completed;
d.Convert(src, nSrc, chOut, nDest, true, out bytesUsed, out charsUsed, out completed);
res = String.Copy(new String(chOut));
}
return(res);
}
internal static unsafe void Convert(this Decoder decoder, ReadOnlySpan <byte> bytes, Span <char> chars, bool flush, out int bytesUsed, out int charsUsed, out bool completed)
{
fixed(byte *bytePtr = &MemoryMarshal.GetReference(bytes))
fixed(char *charPtr = &MemoryMarshal.GetReference(chars))
{
decoder.Convert(bytePtr, bytes.Length, charPtr, chars.Length, flush, out bytesUsed, out charsUsed, out completed);
}
}
internal unsafe string GetString(Span <byte> value)
{
fixed(char *pch = _bufferCharDecoding)
fixed(byte *pb = value)
{
_dec.Convert(pb, value.Length, pch, _bufferCharDecoding.Length, true, out _, out _, out _);
return(new string(pch));
}
}
// Read a string of the specified size
internal static String readString(byte[] input, int offset, int len)
{
int bytesUsed, charsUsed;
bool completed;
char[] outbuf = new char[len];
macdecoder.Convert(input, offset, len, outbuf, 0, len, true,
out bytesUsed, out charsUsed, out completed);
return(new String(outbuf, 0, charsUsed));
}
private void DecodeChar()
{
var c = new char[1];
Decoder.Convert(Buffer, 0, BufferWritePosition, c, 0, 1, true, out var bytesUsed, out var charsUsed, out var completed);
ParseChar(c[0], bytesUsed);
Array.Copy(Buffer, bytesUsed, Buffer, 0, BufferWritePosition - bytesUsed);
BufferWritePosition -= bytesUsed;
}
private static string StreamToString(Stream stream, Encoding encoding)
{
StringBuilder result = new StringBuilder(capacity: ChunkSize);
Decoder decoder = encoding.GetDecoder();
int useBufferSize = 64;
if (useBufferSize < encoding.GetMaxCharCount(10))
{
useBufferSize = encoding.GetMaxCharCount(10);
}
char[] chars = new char[useBufferSize];
byte[] bytes = new byte[useBufferSize * 4];
int bytesRead = 0;
do
{
// Read at most the number of bytes that will fit in the input buffer. The
// return value is the actual number of bytes read, or zero if no bytes remain.
bytesRead = stream.Read(bytes, 0, useBufferSize * 4);
bool completed = false;
int byteIndex = 0;
int bytesUsed;
int charsUsed;
while (!completed)
{
// If this is the last input data, flush the decoder's internal buffer and state.
bool flush = (bytesRead == 0);
decoder.Convert(bytes, byteIndex, bytesRead - byteIndex,
chars, 0, useBufferSize, flush,
out bytesUsed, out charsUsed, out completed);
// The conversion produced the number of characters indicated by charsUsed. Write that number
// of characters to our result buffer
result.Append(chars, 0, charsUsed);
// Increment byteIndex to the next block of bytes in the input buffer, if any, to convert.
byteIndex += bytesUsed;
// The behavior of decoder.Convert changed start .NET 3.1-preview2.
// The change was made in https://github.com/dotnet/coreclr/pull/27229
// The recommendation from .NET team is to not check for 'completed' if 'flush' is false.
// Break out of the loop if all bytes have been read.
if (!flush && bytesRead == byteIndex)
{
break;
}
}
} while (bytesRead != 0);
return(result.ToString());
}
private void VerificationHelper(Decoder decoder, byte[] bytes,
int byteIndex,
int byteCount,
char[] chars,
int charIndex,
int charCount,
bool flush,
int desiredBytesUsed,
int desiredCharsUsed,
bool desiredCompleted,
string errorno)
{
int bytesUsed;
int charsUsed;
bool completed;
decoder.Convert(bytes, byteIndex, byteCount, chars, charIndex, charCount, flush, out bytesUsed,
out charsUsed, out completed);
Assert.Equal(desiredBytesUsed, bytesUsed);
Assert.Equal(desiredCharsUsed, charsUsed);
Assert.Equal(desiredCompleted, completed);
}
public static void Decoder_Convert_Invalid(Encoding encoding, Decoder decoder, bool flush)
{
int bytesUsed = 0;
int charsUsed = 0;
bool completed = false;
Action verifyOutParams = () =>
{
Assert.Equal(0, bytesUsed);
Assert.Equal(0, charsUsed);
Assert.False(completed);
};
// Bytes is null
Assert.Throws<ArgumentNullException>("bytes", () => decoder.Convert(null, 0, 0, new char[4], 0, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
// ByteIndex is invalid
Assert.Throws<ArgumentOutOfRangeException>("byteIndex", () => decoder.Convert(new byte[4], -1, 0, new char[4], 0, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
Assert.Throws<ArgumentOutOfRangeException>("bytes", () => decoder.Convert(new byte[4], 5, 0, new char[4], 0, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
// ByteCount is invalid
Assert.Throws<ArgumentOutOfRangeException>("byteCount", () => decoder.Convert(new byte[4], 0, -1, new char[4], 0, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
Assert.Throws<ArgumentOutOfRangeException>("bytes", () => decoder.Convert(new byte[4], 0, 5, new char[4], 0, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
Assert.Throws<ArgumentOutOfRangeException>("bytes", () => decoder.Convert(new byte[4], 1, 4, new char[4], 0, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
// Chars is null
Assert.Throws<ArgumentNullException>("chars", () => decoder.Convert(new byte[1], 0, 1, null, 0, 0, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
// CharIndex is invalid
Assert.Throws<ArgumentOutOfRangeException>("charIndex", () => decoder.Convert(new byte[1], 0, 0, new char[4], -1, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
Assert.Throws<ArgumentOutOfRangeException>("chars", () => decoder.Convert(new byte[1], 0, 0, new char[4], 5, 0, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
// CharCount is invalid
Assert.Throws<ArgumentOutOfRangeException>("charCount", () => decoder.Convert(new byte[1], 0, 0, new char[4], 0, -1, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
Assert.Throws<ArgumentOutOfRangeException>("chars", () => decoder.Convert(new byte[1], 0, 0, new char[4], 0, 5, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
Assert.Throws<ArgumentOutOfRangeException>("chars", () => decoder.Convert(new byte[1], 0, 0, new char[4], 1, 4, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
// Chars does not have enough space
Assert.Throws<ArgumentException>("chars", () => decoder.Convert(new byte[4], 0, 4, new char[0], 0, 0, flush, out charsUsed, out bytesUsed, out completed));
verifyOutParams();
}