[System.Security.SecurityCritical] // auto-generated
internal unsafe EncodingCharBuffer(EncodingNLS enc, DecoderNLS decoder, char* charStart, int charCount, byte* byteStart, int byteCount)
{
_enc = enc;
_decoder = decoder;
_chars = charStart;
_charStart = charStart;
_charEnd = charStart + charCount;
_byteStart = byteStart;
_bytes = byteStart;
_byteEnd = byteStart + byteCount;
if (_decoder == null)
_fallbackBuffer = enc.DecoderFallback.CreateFallbackBuffer();
else
_fallbackBuffer = _decoder.FallbackBuffer;
// If we're getting chars or getting char count we don't expect to have
// to remember fallbacks between calls (so it should be empty)
Debug.Assert(_fallbackBuffer.Remaining == 0,
"[Encoding.EncodingCharBuffer.EncodingCharBuffer]Expected empty fallback buffer for getchars/charcount");
_fallbackBufferHelper = new DecoderFallbackBufferHelper(_fallbackBuffer);
_fallbackBufferHelper.InternalInitialize(_bytes, _charEnd);
}
private unsafe bool FallbackInvalidByteSequence(ref byte* pSrc, int ch, DecoderFallbackBuffer fallback, ref char* pTarget)
{
byte* numPtr = pSrc;
byte[] bytesUnknown = this.GetBytesUnknown(ref numPtr, ch);
if (!fallback.InternalFallback(bytesUnknown, pSrc, ref pTarget))
{
pSrc = numPtr;
return false;
}
return true;
}
public DecoderFallbackBufferHelper(DecoderFallbackBuffer fallbackBuffer)
{
_fallbackBuffer = fallbackBuffer;
byteStart = null;
charEnd = null;
}
int GetChars (byte[] bytes, int byteIndex, int byteCount,
char[] chars, int charIndex,
ref DecoderFallbackBuffer buffer)
{
if (bytes == null)
throw new ArgumentNullException ("bytes");
if (chars == null)
throw new ArgumentNullException ("chars");
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array"));
if (byteCount < 0 || byteCount > (bytes.Length - byteIndex))
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array"));
if (charIndex < 0 || charIndex > chars.Length)
throw new ArgumentOutOfRangeException ("charIndex", _("ArgRange_Array"));
if ((chars.Length - charIndex) < byteCount)
throw new ArgumentException (_("Arg_InsufficientSpace"));
int count = byteCount;
while (count-- > 0) {
char c = (char) bytes [byteIndex++];
if (c < '\x80')
chars [charIndex++] = c;
else {
if (buffer == null)
buffer = DecoderFallback.CreateFallbackBuffer ();
buffer.Fallback (bytes, byteIndex);
while (buffer.Remaining > 0)
chars [charIndex++] = buffer.GetNextChar ();
}
}
return byteCount;
}
[System.Security.SecurityCritical] // auto-generated
private unsafe bool FallbackInvalidByteSequence(
ref byte* pSrc, int ch, DecoderFallbackBuffer fallback, ref char* pTarget)
{
// Get our byte[]
byte *pStart = pSrc;
byte[] bytesUnknown = GetBytesUnknown(ref pStart, ch);
// Do the actual fallback
if (!fallback.InternalFallback(bytesUnknown, pSrc, ref pTarget))
{
// Oops, it failed, back up to pStart
pSrc = pStart;
return false;
}
// It worked
return true;
}
public DecoderTestFallbackBuffer (DecoderReplacementFallback fallback, FallbackDelegate fallbackAction)
{
this.fallbackAction = fallbackAction;
buffer = new DecoderReplacementFallbackBuffer (fallback);
}
// Get the characters that result from decoding a byte buffer.
private unsafe static int InternalGetChars (
byte[] bytes, int byteIndex, int byteCount, char[] chars,
int charIndex, ref uint leftOverBits, ref uint leftOverCount,
object provider,
ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
{
// Validate the parameters.
if (bytes == null) {
throw new ArgumentNullException ("bytes");
}
if (chars == null) {
throw new ArgumentNullException ("chars");
}
if (byteIndex < 0 || byteIndex > bytes.Length) {
throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array"));
}
if (byteCount < 0 || byteCount > (bytes.Length - byteIndex)) {
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array"));
}
if (charIndex < 0 || charIndex > chars.Length) {
throw new ArgumentOutOfRangeException ("charIndex", _("ArgRange_Array"));
}
if (charIndex == chars.Length)
return 0;
fixed (char* cptr = chars) {
if (byteCount == 0 || byteIndex == bytes.Length)
return InternalGetChars (null, 0, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush);
// otherwise...
fixed (byte* bptr = bytes)
return InternalGetChars (bptr + byteIndex, byteCount, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush);
}
}
[System.Security.SecurityCritical] // auto-generated
internal unsafe EncodingCharBuffer(Encoding enc, DecoderNLS decoder, char* charStart, int charCount,
byte* byteStart, int byteCount)
{
this.enc = enc;
this.decoder = decoder;
this.chars = charStart;
this.charStart = charStart;
this.charEnd = charStart + charCount;
this.byteStart = byteStart;
this.bytes = byteStart;
this.byteEnd = byteStart + byteCount;
if (this.decoder == null)
this.fallbackBuffer = enc.DecoderFallback.CreateFallbackBuffer();
else
this.fallbackBuffer = this.decoder.FallbackBuffer;
// If we're getting chars or getting char count we don't expect to have
// to remember fallbacks between calls (so it should be empty)
Contract.Assert(fallbackBuffer.Remaining == 0,
"[Encoding.EncodingCharBuffer.EncodingCharBuffer]Expected empty fallback buffer for getchars/charcount");
fallbackBuffer.InternalInitialize(bytes, charEnd);
}
// Internal version of "GetCharCount" which can handle a rolling
// state between multiple calls to this method.
private unsafe static int InternalGetCharCount (
byte[] bytes, int index, int count, uint leftOverBits,
uint leftOverCount, object provider,
ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
{
// Validate the parameters.
if (bytes == null) {
throw new ArgumentNullException ("bytes");
}
if (index < 0 || index > bytes.Length) {
throw new ArgumentOutOfRangeException ("index", _("ArgRange_Array"));
}
if (count < 0 || count > (bytes.Length - index)) {
throw new ArgumentOutOfRangeException ("count", _("ArgRange_Array"));
}
if (count == 0)
return 0;
fixed (byte *bptr = bytes)
return InternalGetCharCount (bptr + index, count,
leftOverBits, leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush);
}
// This is internal and called by something else,
internal override unsafe int GetCharCount(byte *bytes, int count, DecoderNLS decoder)
{
// Just assert, we're called internally so these should be safe, checked already
Debug.Assert(bytes != null, "[ASCIIEncoding.GetCharCount]bytes is null");
Debug.Assert(count >= 0, "[ASCIIEncoding.GetCharCount]byteCount is negative");
// ASCII doesn't do best fit, so don't have to check for it, find out which decoder fallback we're using
DecoderReplacementFallback fallback = null;
if (decoder == null)
{
fallback = this.DecoderFallback as DecoderReplacementFallback;
}
else
{
fallback = decoder.Fallback as DecoderReplacementFallback;
Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer ||
decoder.FallbackBuffer.Remaining == 0,
"[ASCIICodePageEncoding.GetCharCount]Expected empty fallback buffer");
}
if (fallback != null && fallback.MaxCharCount == 1)
{
// Just return length, SBCS stay the same length because they don't map to surrogate
// pairs and we don't have a decoder fallback.
return(count);
}
// Only need decoder fallback buffer if not using default replacement fallback, no best fit for ASCII
DecoderFallbackBuffer fallbackBuffer = null;
// Have to do it the hard way.
// Assume charCount will be == count
int charCount = count;
byte[] byteBuffer = new byte[1];
// Do it our fast way
byte *byteEnd = bytes + count;
// Quick loop
while (bytes < byteEnd)
{
// Faster if don't use *bytes++;
byte b = *bytes;
bytes++;
// If unknown we have to do fallback count
if (b >= 0x80)
{
if (fallbackBuffer == null)
{
if (decoder == null)
{
fallbackBuffer = this.DecoderFallback.CreateFallbackBuffer();
}
else
{
fallbackBuffer = decoder.FallbackBuffer;
}
fallbackBuffer.InternalInitialize(byteEnd - count, null);
}
// Use fallback buffer
byteBuffer[0] = b;
charCount--; // Have to unreserve the one we already allocated for b
charCount += fallbackBuffer.InternalFallback(byteBuffer, bytes);
}
}
// Fallback buffer must be empty
Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0,
"[ASCIIEncoding.GetCharCount]Expected Empty fallback buffer");
// Converted sequence is same length as input
return(charCount);
}
int GetChars (byte[] bytes, int byteIndex, int byteCount,
char[] chars, int charIndex,
ref DecoderFallbackBuffer buffer)
{
if (bytes == null)
throw new ArgumentNullException ("bytes");
if (chars == null)
throw new ArgumentNullException ("chars");
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array"));
if (byteCount < 0 || byteCount > (bytes.Length - byteIndex))
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array"));
if (charIndex < 0 || charIndex > chars.Length)
throw new ArgumentOutOfRangeException ("charIndex", _("ArgRange_Array"));
if ((chars.Length - charIndex) < byteCount)
throw new ArgumentException (_("Arg_InsufficientSpace"));
int count = byteCount;
while (count-- > 0) {
char c = (char) bytes [byteIndex++];
if (c < '\x80')
chars [charIndex++] = c;
else {
if (buffer == null)
buffer = DecoderFallback.CreateFallbackBuffer ();
var thisByte = new byte[] { bytes [byteIndex-1] };
buffer.Fallback (thisByte, 0);
while (buffer.Remaining > 0) {
if (charIndex < chars.Length) {
chars [charIndex++] = buffer.GetNextChar ();
continue;
}
throw new ArgumentException (
"The output char buffer is too small to contain the " +
"decoded characters.");
}
}
}
return byteCount;
}
// This function is called when we want to flush the decoder state
// (i.e. in case of invalid UTF-8 characters or interrupted sequences)
internal unsafe static DecoderStatus InternalGetCharsFlush (
char* chars, int charCount,
DecoderFallbackBuffer fallbackBuffer,
DecoderStatus s,
int bytesProcessed, ref int charsProcessed,
ref uint leftBytes, ref uint leftBits, ref uint procBytes)
{
// if there is nothing to flush, then exit silently
if(procBytes == 0)
return DecoderStatus.Ok;
// now we build a 'bytesUnknown' array with the
// stored bytes in 'procBytes'.
int extra = 0;
for (uint t = procBytes; t != 0; extra++)
t = t >> 8;
byte [] bytesUnknown = new byte [extra];
for (int i = extra; i > 0; i--)
bytesUnknown [i - 1] = (byte) ((procBytes >> (8 * (extra - i))) & 0xff);
// partial reset: this condition avoids infinite loops
if (s == DecoderStatus.InvalidSequence)
leftBytes = 0;
// call the fallback and cross fingers
fallbackBuffer.Fallback (bytesUnknown, bytesProcessed - extra);
if (chars != null) {
while (fallbackBuffer.Remaining > 0) {
if (charsProcessed >= charCount)
return DecoderStatus.InsufficientSpace;
chars [charsProcessed++] = fallbackBuffer.GetNextChar ();
}
} else
charsProcessed += fallbackBuffer.Remaining;
fallbackBuffer.Reset ();
// recovery was succesful, flush decoder state
leftBits = leftBytes = procBytes = 0;
return DecoderStatus.Ok;
}
[System.Security.SecurityCritical] // auto-generated
public override unsafe int GetChars(byte* bytes, int byteCount,
char* chars, int charCount, DecoderNLS decoder)
{
// Just need to ASSERT, this is called by something else internal that checked parameters already
Debug.Assert(bytes != null, "[SBCSCodePageEncoding.GetChars]bytes is null");
Debug.Assert(byteCount >= 0, "[SBCSCodePageEncoding.GetChars]byteCount is negative");
Debug.Assert(chars != null, "[SBCSCodePageEncoding.GetChars]chars is null");
Debug.Assert(charCount >= 0, "[SBCSCodePageEncoding.GetChars]charCount is negative");
CheckMemorySection();
// See if we have best fit
bool bUseBestFit = false;
// Do it fast way if using ? replacement or best fit fallbacks
byte* byteEnd = bytes + byteCount;
byte* byteStart = bytes;
char* charStart = chars;
// Only need decoder fallback buffer if not using default replacement fallback or best fit fallback.
DecoderReplacementFallback fallback = null;
if (decoder == null)
{
fallback = DecoderFallback as DecoderReplacementFallback;
bUseBestFit = DecoderFallback is InternalDecoderBestFitFallback;
}
else
{
fallback = decoder.Fallback as DecoderReplacementFallback;
bUseBestFit = decoder.Fallback is InternalDecoderBestFitFallback;
Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer ||
decoder.FallbackBuffer.Remaining == 0,
"[SBCSCodePageEncoding.GetChars]Expected empty fallback buffer at start");
}
if (bUseBestFit || (fallback != null && fallback.MaxCharCount == 1))
{
// Try it the fast way
char replacementChar;
if (fallback == null)
replacementChar = '?'; // Best fit always has ? for fallback for SBCS
else
replacementChar = fallback.DefaultString[0];
// Need byteCount chars, otherwise too small buffer
if (charCount < byteCount)
{
// Need at least 1 output byte, throw if must throw
ThrowCharsOverflow(decoder, charCount < 1);
// Not throwing, use what we can
byteEnd = bytes + charCount;
}
// Quick loop, just do '?' replacement because we don't have fallbacks for decodings.
while (bytes < byteEnd)
{
char c;
if (bUseBestFit)
{
if (arrayBytesBestFit == null)
{
ReadBestFitTable();
}
c = arrayBytesBestFit[*bytes];
}
else
c = _mapBytesToUnicode[*bytes];
bytes++;
if (c == UNKNOWN_CHAR)
// This is an invalid byte in the ASCII encoding.
*chars = replacementChar;
else
*chars = c;
chars++;
}
// bytes & chars used are the same
if (decoder != null)
decoder.m_bytesUsed = (int)(bytes - byteStart);
return (int)(chars - charStart);
}
// Slower way's going to need a fallback buffer
DecoderFallbackBuffer fallbackBuffer = null;
byte[] byteBuffer = new byte[1];
char* charEnd = chars + charCount;
DecoderFallbackBufferHelper fallbackHelper = new DecoderFallbackBufferHelper(decoder.FallbackBuffer);
// Not quite so fast loop
while (bytes < byteEnd)
{
// Faster if don't use *bytes++;
char c = _mapBytesToUnicode[*bytes];
bytes++;
// See if it was unknown
if (c == UNKNOWN_CHAR)
{
// Make sure we have a fallback buffer
if (fallbackBuffer == null)
{
if (decoder == null)
fallbackBuffer = DecoderFallback.CreateFallbackBuffer();
else
fallbackBuffer = decoder.FallbackBuffer;
fallbackHelper = new DecoderFallbackBufferHelper(fallbackBuffer);
fallbackHelper.InternalInitialize(byteEnd - byteCount, charEnd);
}
// Use fallback buffer
Debug.Assert(bytes > byteStart,
"[SBCSCodePageEncoding.GetChars]Expected bytes to have advanced already (unknown byte)");
byteBuffer[0] = *(bytes - 1);
// Fallback adds fallback to chars, but doesn't increment chars unless the whole thing fits.
if (!fallbackHelper.InternalFallback(byteBuffer, bytes, ref chars))
{
// May or may not throw, but we didn't get this byte
bytes--; // unused byte
fallbackHelper.InternalReset(); // Didn't fall this back
ThrowCharsOverflow(decoder, bytes == byteStart); // throw?
break; // don't throw, but stop loop
}
}
else
{
// Make sure we have buffer space
if (chars >= charEnd)
{
Debug.Assert(bytes > byteStart,
"[SBCSCodePageEncoding.GetChars]Expected bytes to have advanced already (known byte)");
bytes--; // unused byte
ThrowCharsOverflow(decoder, bytes == byteStart); // throw?
break; // don't throw, but stop loop
}
*(chars) = c;
chars++;
}
}
// Might have had decoder fallback stuff.
if (decoder != null)
decoder.m_bytesUsed = (int)(bytes - byteStart);
// Expect Empty fallback buffer for GetChars
Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0,
"[SBCSEncoding.GetChars]Expected Empty fallback buffer at end");
return (int)(chars - charStart);
}
[System.Security.SecurityCritical] // auto-generated
public override unsafe int GetCharCount(byte* bytes, int count, DecoderNLS decoder)
{
// Just assert, we're called internally so these should be safe, checked already
Debug.Assert(bytes != null, "[SBCSCodePageEncoding.GetCharCount]bytes is null");
Debug.Assert(count >= 0, "[SBCSCodePageEncoding.GetCharCount]byteCount is negative");
CheckMemorySection();
// See if we have best fit
bool bUseBestFit = false;
// Only need decoder fallback buffer if not using default replacement fallback or best fit fallback.
DecoderReplacementFallback fallback = null;
if (decoder == null)
{
fallback = DecoderFallback as DecoderReplacementFallback;
bUseBestFit = DecoderFallback is InternalDecoderBestFitFallback;
}
else
{
fallback = decoder.Fallback as DecoderReplacementFallback;
bUseBestFit = decoder.Fallback is InternalDecoderBestFitFallback;
Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer ||
decoder.FallbackBuffer.Remaining == 0,
"[SBCSCodePageEncoding.GetChars]Expected empty fallback buffer at start");
}
if (bUseBestFit || (fallback != null && fallback.MaxCharCount == 1))
{
// Just return length, SBCS stay the same length because they don't map to surrogate
// pairs and we don't have a decoder fallback.
return count;
}
// Might need one of these later
DecoderFallbackBuffer fallbackBuffer = null;
DecoderFallbackBufferHelper fallbackHelper = new DecoderFallbackBufferHelper(fallbackBuffer);
// Have to do it the hard way.
// Assume charCount will be == count
int charCount = count;
byte[] byteBuffer = new byte[1];
// Do it our fast way
byte* byteEnd = bytes + count;
// Quick loop
while (bytes < byteEnd)
{
// Faster if don't use *bytes++;
char c;
c = _mapBytesToUnicode[*bytes];
bytes++;
// If unknown we have to do fallback count
if (c == UNKNOWN_CHAR)
{
// Must have a fallback buffer
if (fallbackBuffer == null)
{
// Need to adjust count so we get real start
if (decoder == null)
fallbackBuffer = DecoderFallback.CreateFallbackBuffer();
else
fallbackBuffer = decoder.FallbackBuffer;
fallbackHelper = new DecoderFallbackBufferHelper(fallbackBuffer);
fallbackHelper.InternalInitialize(byteEnd - count, null);
}
// Use fallback buffer
byteBuffer[0] = *(bytes - 1);
charCount--; // We'd already reserved one for *(bytes-1)
charCount += fallbackHelper.InternalFallback(byteBuffer, bytes);
}
}
// Fallback buffer must be empty
Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0,
"[SBCSEncoding.GetCharCount]Expected Empty fallback buffer at end");
// Converted sequence is same length as input
return charCount;
}
internal override unsafe int GetChars(byte *bytes, int byteCount,
char *chars, int charCount, DecoderNLS decoder)
{
// Just need to ASSERT, this is called by something else internal that checked parameters already
Debug.Assert(bytes != null, "[ASCIIEncoding.GetChars]bytes is null");
Debug.Assert(byteCount >= 0, "[ASCIIEncoding.GetChars]byteCount is negative");
Debug.Assert(chars != null, "[ASCIIEncoding.GetChars]chars is null");
Debug.Assert(charCount >= 0, "[ASCIIEncoding.GetChars]charCount is negative");
// Do it fast way if using ? replacement fallback
byte *byteEnd = bytes + byteCount;
byte *byteStart = bytes;
char *charStart = chars;
// Note: ASCII doesn't do best fit, but we have to fallback if they use something > 0x7f
// Only need decoder fallback buffer if not using ? fallback.
// ASCII doesn't do best fit, so don't have to check for it, find out which decoder fallback we're using
DecoderReplacementFallback fallback = null;
if (decoder == null)
{
fallback = this.DecoderFallback as DecoderReplacementFallback;
}
else
{
fallback = decoder.Fallback as DecoderReplacementFallback;
Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer ||
decoder.FallbackBuffer.Remaining == 0,
"[ASCIICodePageEncoding.GetChars]Expected empty fallback buffer");
}
if (fallback != null && fallback.MaxCharCount == 1)
{
// Try it the fast way
char replacementChar = fallback.DefaultString[0];
// Need byteCount chars, otherwise too small buffer
if (charCount < byteCount)
{
// Need at least 1 output byte, throw if must throw
ThrowCharsOverflow(decoder, charCount < 1);
// Not throwing, use what we can
byteEnd = bytes + charCount;
}
// Quick loop, just do '?' replacement because we don't have fallbacks for decodings.
while (bytes < byteEnd)
{
byte b = *(bytes++);
if (b >= 0x80)
{
// This is an invalid byte in the ASCII encoding.
*(chars++) = replacementChar;
}
else
{
*(chars++) = unchecked ((char)b);
}
}
// bytes & chars used are the same
if (decoder != null)
{
decoder.m_bytesUsed = (int)(bytes - byteStart);
}
return((int)(chars - charStart));
}
// Slower way's going to need a fallback buffer
DecoderFallbackBuffer fallbackBuffer = null;
byte[] byteBuffer = new byte[1];
char * charEnd = chars + charCount;
// Not quite so fast loop
while (bytes < byteEnd)
{
// Faster if don't use *bytes++;
byte b = *(bytes);
bytes++;
if (b >= 0x80)
{
// This is an invalid byte in the ASCII encoding.
if (fallbackBuffer == null)
{
if (decoder == null)
{
fallbackBuffer = this.DecoderFallback.CreateFallbackBuffer();
}
else
{
fallbackBuffer = decoder.FallbackBuffer;
}
fallbackBuffer.InternalInitialize(byteEnd - byteCount, charEnd);
}
// Use fallback buffer
byteBuffer[0] = b;
// Note that chars won't get updated unless this succeeds
if (!fallbackBuffer.InternalFallback(byteBuffer, bytes, ref chars))
{
// May or may not throw, but we didn't get this byte
Debug.Assert(bytes > byteStart || chars == charStart,
"[ASCIIEncoding.GetChars]Expected bytes to have advanced already (fallback case)");
bytes--; // unused byte
fallbackBuffer.InternalReset(); // Didn't fall this back
ThrowCharsOverflow(decoder, chars == charStart); // throw?
break; // don't throw, but stop loop
}
}
else
{
// Make sure we have buffer space
if (chars >= charEnd)
{
Debug.Assert(bytes > byteStart || chars == charStart,
"[ASCIIEncoding.GetChars]Expected bytes to have advanced already (normal case)");
bytes--; // unused byte
ThrowCharsOverflow(decoder, chars == charStart); // throw?
break; // don't throw, but stop loop
}
*(chars) = unchecked ((char)b);
chars++;
}
}
// Might have had decoder fallback stuff.
if (decoder != null)
{
decoder.m_bytesUsed = (int)(bytes - byteStart);
}
// Expect Empty fallback buffer for GetChars
Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0,
"[ASCIIEncoding.GetChars]Expected Empty fallback buffer");
return((int)(chars - charStart));
}
private unsafe int FallbackInvalidByteSequence(byte* pSrc, int ch, DecoderFallbackBuffer fallback)
{
byte[] bytesUnknown = this.GetBytesUnknown(ref pSrc, ch);
return fallback.InternalFallback(bytesUnknown, pSrc);
}
// for GetCharCount()
static unsafe int Fallback (object provider, ref DecoderFallbackBuffer buffer, ref byte [] bufferArg, byte* bytes, long index, uint size)
{
if (buffer == null) {
DecoderFallback fb = provider as DecoderFallback;
if (fb != null)
buffer = fb.CreateFallbackBuffer ();
else
buffer = ((Decoder) provider).FallbackBuffer;
}
if (bufferArg == null)
bufferArg = new byte [1];
int ret = 0;
for (int i = 0; i < size; i++) {
bufferArg [0] = bytes [(int) index + i];
buffer.Fallback (bufferArg, 0);
ret += buffer.Remaining;
buffer.Reset ();
}
return ret;
}
// InternalGetChars processor. Can decode or count space needed for
// decoding, depending on the enabled mode:
// - decoder
// enabled when charCount >= 0 (but chars may be null)
// - counter
// enabled when chars == null && charCount < 0
internal unsafe static DecoderStatus InternalGetChars (
byte* bytes, int byteCount,
char* chars, int charCount,
DecoderFallbackBuffer fallbackBuffer,
out int bytesProcessed, out int charsProcessed,
ref uint leftBytes, ref uint leftBits, ref uint procBytes,
bool flush)
{
DecoderStatus s;
int t_bytesProcessed, t_charsProcessed;
// Validate parameters
if (byteCount < 0)
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_NonNegative"));
else
if (byteCount > 0 && bytes == null)
throw new ArgumentNullException ("bytes");
if (chars == null) {
if (charCount > 0)
throw new ArgumentNullException ("chars");
} else {
if (charCount < 0)
throw new ArgumentOutOfRangeException ("charCount", _("ArgRange_NonNegative"));
}
// reset counters
charsProcessed = 0;
bytesProcessed = 0;
// byte processing loop
while (byteCount - bytesProcessed > 0) {
// fetch a char from the input byte array
s = chars != null
? InternalGetChar (
bytes + bytesProcessed, byteCount - bytesProcessed,
chars + charsProcessed, charCount - charsProcessed,
out t_bytesProcessed, out t_charsProcessed,
ref leftBytes, ref leftBits, ref procBytes)
: InternalGetChar (
bytes + bytesProcessed, byteCount - bytesProcessed,
null, charCount,
out t_bytesProcessed, out t_charsProcessed,
ref leftBytes, ref leftBits, ref procBytes);
// if not enough space return here
// NOTE: maybe we should restore the original encoder
// state ... we should check what ms do in this case
if(s == DecoderStatus.InsufficientSpace)
return DecoderStatus.InsufficientSpace;
// update counters
charsProcessed += t_charsProcessed;
bytesProcessed += t_bytesProcessed;
switch (s) {
case DecoderStatus.Ok:
break; // everything OK :D
case DecoderStatus.Overlong:
case DecoderStatus.InvalidSequence:
case DecoderStatus.InvalidStart:
case DecoderStatus.InvalidChar:
case DecoderStatus.SurrogateFound:
s = InternalGetCharsFlush (
chars, charCount,
fallbackBuffer,
s,
bytesProcessed, ref charsProcessed,
ref leftBytes, ref leftBits, ref procBytes);
if (s != DecoderStatus.Ok)
return s;
break;
case DecoderStatus.InputRunOut:
return flush
? InternalGetCharsFlush (
chars, charCount,
fallbackBuffer,
s,
bytesProcessed, ref charsProcessed,
ref leftBytes, ref leftBits, ref procBytes)
: DecoderStatus.InputRunOut;
}
}
return flush
? InternalGetCharsFlush (
chars, charCount,
fallbackBuffer,
DecoderStatus.Ok,
bytesProcessed, ref charsProcessed,
ref leftBytes, ref leftBits, ref procBytes)
: DecoderStatus.Ok;
}
public DecoderFallbackBufferHelper(DecoderFallbackBuffer fallbackBuffer)
{
_fallbackBuffer = fallbackBuffer;
byteStart = null;
charEnd = null;
}
private unsafe static int InternalGetCharCount (
byte* bytes, int count, uint leftOverBits,
uint leftOverCount, object provider,
ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
{
int index = 0;
int length = 0;
if (leftOverCount == 0) {
int end = index + count;
for (; index < end; index++, count--) {
if (bytes [index] < 0x80)
length++;
else
break;
}
}
// Determine the number of characters that we have.
uint ch;
uint leftBits = leftOverBits;
uint leftSoFar = (leftOverCount & (uint)0x0F);
uint leftSize = ((leftOverCount >> 4) & (uint)0x0F);
while (count > 0) {
ch = (uint)(bytes[index++]);
--count;
if (leftSize == 0) {
// Process a UTF-8 start character.
if (ch < (uint)0x0080) {
// Single-byte UTF-8 character.
++length;
} else if ((ch & (uint)0xE0) == (uint)0xC0) {
// Double-byte UTF-8 character.
leftBits = (ch & (uint)0x1F);
leftSoFar = 1;
leftSize = 2;
} else if ((ch & (uint)0xF0) == (uint)0xE0) {
// Three-byte UTF-8 character.
leftBits = (ch & (uint)0x0F);
leftSoFar = 1;
leftSize = 3;
} else if ((ch & (uint)0xF8) == (uint)0xF0) {
// Four-byte UTF-8 character.
leftBits = (ch & (uint)0x07);
leftSoFar = 1;
leftSize = 4;
} else if ((ch & (uint)0xFC) == (uint)0xF8) {
// Five-byte UTF-8 character.
leftBits = (ch & (uint)0x03);
leftSoFar = 1;
leftSize = 5;
} else if ((ch & (uint)0xFE) == (uint)0xFC) {
// Six-byte UTF-8 character.
leftBits = (ch & (uint)0x03);
leftSoFar = 1;
leftSize = 6;
} else {
// Invalid UTF-8 start character.
length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - 1, 1);
}
} else {
// Process an extra byte in a multi-byte sequence.
if ((ch & (uint)0xC0) == (uint)0x80) {
leftBits = ((leftBits << 6) | (ch & (uint)0x3F));
if (++leftSoFar >= leftSize) {
// We have a complete character now.
if (leftBits < (uint)0x10000) {
// is it an overlong ?
bool overlong = false;
switch (leftSize) {
case 2:
overlong = (leftBits <= 0x7F);
break;
case 3:
overlong = (leftBits <= 0x07FF);
break;
case 4:
overlong = (leftBits <= 0xFFFF);
break;
case 5:
overlong = (leftBits <= 0x1FFFFF);
break;
case 6:
overlong = (leftBits <= 0x03FFFFFF);
break;
}
if (overlong) {
length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
}
else if ((leftBits & 0xF800) == 0xD800) {
// UTF-8 doesn't use surrogate characters
length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
}
else
++length;
} else if (leftBits < (uint)0x110000) {
length += 2;
} else {
length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
}
leftSize = 0;
}
} else {
// Invalid UTF-8 sequence: clear and restart.
length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
leftSize = 0;
--index;
++count;
}
}
}
if (flush && leftSize != 0) {
// We had left-over bytes that didn't make up
// a complete UTF-8 character sequence.
length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
}
// Return the final length to the caller.
return length;
}
internal unsafe static DecoderStatus InternalGetCharsCount (
byte* bytes, int byteCount,
DecoderFallbackBuffer fallbackBuffer,
out int bytesProcessed, out int charsProcessed,
ref uint leftBytes, ref uint leftBits, ref uint procBytes,
bool flush)
{
if (byteCount < 0)
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array"));
return InternalGetChars (
bytes, byteCount,
null, -1,
fallbackBuffer,
out bytesProcessed, out charsProcessed,
ref leftBytes, ref leftBits, ref procBytes,
flush);
}
// for GetChars()
static unsafe void Fallback (object provider, ref DecoderFallbackBuffer buffer, ref byte [] bufferArg, byte* bytes, long byteIndex, uint size,
char* chars, ref int charIndex)
{
if (buffer == null) {
DecoderFallback fb = provider as DecoderFallback;
if (fb != null)
buffer = fb.CreateFallbackBuffer ();
else
buffer = ((Decoder) provider).FallbackBuffer;
}
if (bufferArg == null)
bufferArg = new byte [1];
for (int i = 0; i < size; i++) {
bufferArg [0] = bytes [byteIndex + i];
buffer.Fallback (bufferArg, 0);
while (buffer.Remaining > 0)
chars [charIndex++] = buffer.GetNextChar ();
buffer.Reset ();
}
}
internal unsafe static DecoderStatus InternalGetCharsCount (
byte[] bytes, int byteIndex, int byteCount,
DecoderFallbackBuffer fallbackBuffer,
out int bytesProcessed, out int charsProcessed,
ref uint leftBytes, ref uint leftBits, ref uint procBytes,
bool flush)
{
if (bytes == null)
throw new ArgumentNullException ("bytes");
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array"));
if (byteCount < 0 || byteCount > (bytes.Length - byteIndex))
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array"));
fixed (byte* bptr = bytes) {
return InternalGetChars (
bptr + byteIndex, byteCount,
null, -1,
fallbackBuffer,
out bytesProcessed, out charsProcessed,
ref leftBytes, ref leftBits, ref procBytes,
flush);
}
}
private unsafe static int InternalGetChars (
byte* bytes, int byteCount, char* chars, int charCount,
ref uint leftOverBits, ref uint leftOverCount,
object provider,
ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
{
int charIndex = 0, byteIndex = 0;
int length = charCount;
int posn = charIndex;
if (leftOverCount == 0) {
int end = byteIndex + byteCount;
for (; byteIndex < end; posn++, byteIndex++, byteCount--) {
if (bytes [byteIndex] < 0x80)
chars [posn] = (char) bytes [byteIndex];
else
break;
}
}
// Convert the bytes into the output buffer.
uint ch;
uint leftBits = leftOverBits;
uint leftSoFar = (leftOverCount & (uint)0x0F);
uint leftSize = ((leftOverCount >> 4) & (uint)0x0F);
int byteEnd = byteIndex + byteCount;
for(; byteIndex < byteEnd; byteIndex++) {
// Fetch the next character from the byte buffer.
ch = (uint)(bytes[byteIndex]);
if (leftSize == 0) {
// Process a UTF-8 start character.
if (ch < (uint)0x0080) {
// Single-byte UTF-8 character.
if (posn >= length) {
throw new ArgumentException (_("Arg_InsufficientSpace"), "chars");
}
chars[posn++] = (char)ch;
} else if ((ch & (uint)0xE0) == (uint)0xC0) {
// Double-byte UTF-8 character.
leftBits = (ch & (uint)0x1F);
leftSoFar = 1;
leftSize = 2;
} else if ((ch & (uint)0xF0) == (uint)0xE0) {
// Three-byte UTF-8 character.
leftBits = (ch & (uint)0x0F);
leftSoFar = 1;
leftSize = 3;
} else if ((ch & (uint)0xF8) == (uint)0xF0) {
// Four-byte UTF-8 character.
leftBits = (ch & (uint)0x07);
leftSoFar = 1;
leftSize = 4;
} else if ((ch & (uint)0xFC) == (uint)0xF8) {
// Five-byte UTF-8 character.
leftBits = (ch & (uint)0x03);
leftSoFar = 1;
leftSize = 5;
} else if ((ch & (uint)0xFE) == (uint)0xFC) {
// Six-byte UTF-8 character.
leftBits = (ch & (uint)0x03);
leftSoFar = 1;
leftSize = 6;
} else {
// Invalid UTF-8 start character.
Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex, 1, chars, ref posn);
}
} else {
// Process an extra byte in a multi-byte sequence.
if ((ch & (uint)0xC0) == (uint)0x80) {
leftBits = ((leftBits << 6) | (ch & (uint)0x3F));
if (++leftSoFar >= leftSize) {
// We have a complete character now.
if (leftBits < (uint)0x10000) {
// is it an overlong ?
bool overlong = false;
switch (leftSize) {
case 2:
overlong = (leftBits <= 0x7F);
break;
case 3:
overlong = (leftBits <= 0x07FF);
break;
case 4:
overlong = (leftBits <= 0xFFFF);
break;
case 5:
overlong = (leftBits <= 0x1FFFFF);
break;
case 6:
overlong = (leftBits <= 0x03FFFFFF);
break;
}
if (overlong) {
Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
}
else if ((leftBits & 0xF800) == 0xD800) {
// UTF-8 doesn't use surrogate characters
Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
}
else {
if (posn >= length) {
throw new ArgumentException
(_("Arg_InsufficientSpace"), "chars");
}
chars[posn++] = (char)leftBits;
}
} else if (leftBits < (uint)0x110000) {
if ((posn + 2) > length) {
throw new ArgumentException
(_("Arg_InsufficientSpace"), "chars");
}
leftBits -= (uint)0x10000;
chars[posn++] = (char)((leftBits >> 10) +
(uint)0xD800);
chars[posn++] =
(char)((leftBits & (uint)0x3FF) + (uint)0xDC00);
} else {
Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
}
leftSize = 0;
}
} else {
// Invalid UTF-8 sequence: clear and restart.
Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
leftSize = 0;
--byteIndex;
}
}
}
if (flush && leftSize != 0) {
// We had left-over bytes that didn't make up
// a complete UTF-8 character sequence.
Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
}
leftOverBits = leftBits;
leftOverCount = (leftSoFar | (leftSize << 4));
// Return the final length to the caller.
return posn - charIndex;
}
// Internal version of "GetCharCount" which can handle a rolling // state between multiple calls to this method. #if NET_2_0 private unsafe static int InternalGetCharCount ( byte[] bytes, int index, int count, uint leftOverBits, uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
internal unsafe EncodingCharBuffer(Encoding enc, DecoderNLS decoder, char* charStart, int charCount, byte* byteStart, int byteCount)
{
this.enc = enc;
this.decoder = decoder;
this.chars = charStart;
this.charStart = charStart;
this.charEnd = charStart + charCount;
this.byteStart = byteStart;
this.bytes = byteStart;
this.byteEnd = byteStart + byteCount;
if (this.decoder == null)
{
this.fallbackBuffer = enc.DecoderFallback.CreateFallbackBuffer();
}
else
{
this.fallbackBuffer = this.decoder.FallbackBuffer;
}
this.fallbackBuffer.InternalInitialize(this.bytes, this.charEnd);
}
private unsafe static int InternalGetChars ( byte* bytes, int byteCount, char* chars, int charCount, ref uint leftOverBits, ref uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
[System.Security.SecurityCritical] // auto-generated
private unsafe int FallbackInvalidByteSequence(
byte* pSrc, int ch, DecoderFallbackBuffer fallback)
{
// Get our byte[]
byte[] bytesUnknown = GetBytesUnknown(ref pSrc, ch);
// Do the actual fallback
int count = fallback.InternalFallback(bytesUnknown, pSrc);
// # of fallback chars expected.
// Note that we only get here for "long" sequences, and have already unreserved
// the count that we prereserved for the input bytes
return count;
}
int GetChars(byte[] bytes, int byteIndex, int byteCount,
char[] chars, int charIndex,
ref DecoderFallbackBuffer buffer)
{
if (bytes == null)
{
throw new ArgumentNullException("bytes");
}
if (chars == null)
{
throw new ArgumentNullException("chars");
}
if (byteIndex < 0 || byteIndex > bytes.Length)
{
throw new ArgumentOutOfRangeException("byteIndex", _("ArgRange_Array"));
}
if (byteCount < 0 || byteCount > (bytes.Length - byteIndex))
{
throw new ArgumentOutOfRangeException("byteCount", _("ArgRange_Array"));
}
if (charIndex < 0 || charIndex > chars.Length)
{
throw new ArgumentOutOfRangeException("charIndex", _("ArgRange_Array"));
}
if ((chars.Length - charIndex) < byteCount)
{
throw new ArgumentException(_("Arg_InsufficientSpace"));
}
int count = byteCount;
while (count-- > 0)
{
char c = (char)bytes [byteIndex++];
if (c < '\x80')
{
chars [charIndex++] = c;
}
else
{
if (buffer == null)
{
buffer = DecoderFallback.CreateFallbackBuffer();
}
var thisByte = new byte[] { bytes [byteIndex - 1] };
buffer.Fallback(thisByte, 0);
while (buffer.Remaining > 0)
{
if (charIndex < chars.Length)
{
chars [charIndex++] = buffer.GetNextChar();
continue;
}
throw new ArgumentException(
"The output char buffer is too small to contain the " +
"decoded characters.");
}
}
}
return(byteCount);
}