[System.Security.SecurityCritical] // auto-generated internal unsafe EncodingCharBuffer(EncodingNLS enc, DecoderNLS decoder, char* charStart, int charCount, byte* byteStart, int byteCount) { _enc = enc; _decoder = decoder; _chars = charStart; _charStart = charStart; _charEnd = charStart + charCount; _byteStart = byteStart; _bytes = byteStart; _byteEnd = byteStart + byteCount; if (_decoder == null) _fallbackBuffer = enc.DecoderFallback.CreateFallbackBuffer(); else _fallbackBuffer = _decoder.FallbackBuffer; // If we're getting chars or getting char count we don't expect to have // to remember fallbacks between calls (so it should be empty) Debug.Assert(_fallbackBuffer.Remaining == 0, "[Encoding.EncodingCharBuffer.EncodingCharBuffer]Expected empty fallback buffer for getchars/charcount"); _fallbackBufferHelper = new DecoderFallbackBufferHelper(_fallbackBuffer); _fallbackBufferHelper.InternalInitialize(_bytes, _charEnd); }
private unsafe bool FallbackInvalidByteSequence(ref byte* pSrc, int ch, DecoderFallbackBuffer fallback, ref char* pTarget) { byte* numPtr = pSrc; byte[] bytesUnknown = this.GetBytesUnknown(ref numPtr, ch); if (!fallback.InternalFallback(bytesUnknown, pSrc, ref pTarget)) { pSrc = numPtr; return false; } return true; }
public DecoderFallbackBufferHelper(DecoderFallbackBuffer fallbackBuffer) { _fallbackBuffer = fallbackBuffer; byteStart = null; charEnd = null; }
int GetChars (byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex, ref DecoderFallbackBuffer buffer) { if (bytes == null) throw new ArgumentNullException ("bytes"); if (chars == null) throw new ArgumentNullException ("chars"); if (byteIndex < 0 || byteIndex > bytes.Length) throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array")); if (byteCount < 0 || byteCount > (bytes.Length - byteIndex)) throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array")); if (charIndex < 0 || charIndex > chars.Length) throw new ArgumentOutOfRangeException ("charIndex", _("ArgRange_Array")); if ((chars.Length - charIndex) < byteCount) throw new ArgumentException (_("Arg_InsufficientSpace")); int count = byteCount; while (count-- > 0) { char c = (char) bytes [byteIndex++]; if (c < '\x80') chars [charIndex++] = c; else { if (buffer == null) buffer = DecoderFallback.CreateFallbackBuffer (); buffer.Fallback (bytes, byteIndex); while (buffer.Remaining > 0) chars [charIndex++] = buffer.GetNextChar (); } } return byteCount; }
[System.Security.SecurityCritical] // auto-generated private unsafe bool FallbackInvalidByteSequence( ref byte* pSrc, int ch, DecoderFallbackBuffer fallback, ref char* pTarget) { // Get our byte[] byte *pStart = pSrc; byte[] bytesUnknown = GetBytesUnknown(ref pStart, ch); // Do the actual fallback if (!fallback.InternalFallback(bytesUnknown, pSrc, ref pTarget)) { // Oops, it failed, back up to pStart pSrc = pStart; return false; } // It worked return true; }
public DecoderTestFallbackBuffer (DecoderReplacementFallback fallback, FallbackDelegate fallbackAction) { this.fallbackAction = fallbackAction; buffer = new DecoderReplacementFallbackBuffer (fallback); }
// Get the characters that result from decoding a byte buffer. private unsafe static int InternalGetChars ( byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex, ref uint leftOverBits, ref uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush) { // Validate the parameters. if (bytes == null) { throw new ArgumentNullException ("bytes"); } if (chars == null) { throw new ArgumentNullException ("chars"); } if (byteIndex < 0 || byteIndex > bytes.Length) { throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array")); } if (byteCount < 0 || byteCount > (bytes.Length - byteIndex)) { throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array")); } if (charIndex < 0 || charIndex > chars.Length) { throw new ArgumentOutOfRangeException ("charIndex", _("ArgRange_Array")); } if (charIndex == chars.Length) return 0; fixed (char* cptr = chars) { if (byteCount == 0 || byteIndex == bytes.Length) return InternalGetChars (null, 0, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush); // otherwise... fixed (byte* bptr = bytes) return InternalGetChars (bptr + byteIndex, byteCount, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush); } }
[System.Security.SecurityCritical] // auto-generated internal unsafe EncodingCharBuffer(Encoding enc, DecoderNLS decoder, char* charStart, int charCount, byte* byteStart, int byteCount) { this.enc = enc; this.decoder = decoder; this.chars = charStart; this.charStart = charStart; this.charEnd = charStart + charCount; this.byteStart = byteStart; this.bytes = byteStart; this.byteEnd = byteStart + byteCount; if (this.decoder == null) this.fallbackBuffer = enc.DecoderFallback.CreateFallbackBuffer(); else this.fallbackBuffer = this.decoder.FallbackBuffer; // If we're getting chars or getting char count we don't expect to have // to remember fallbacks between calls (so it should be empty) Contract.Assert(fallbackBuffer.Remaining == 0, "[Encoding.EncodingCharBuffer.EncodingCharBuffer]Expected empty fallback buffer for getchars/charcount"); fallbackBuffer.InternalInitialize(bytes, charEnd); }
// Internal version of "GetCharCount" which can handle a rolling // state between multiple calls to this method. private unsafe static int InternalGetCharCount ( byte[] bytes, int index, int count, uint leftOverBits, uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush) { // Validate the parameters. if (bytes == null) { throw new ArgumentNullException ("bytes"); } if (index < 0 || index > bytes.Length) { throw new ArgumentOutOfRangeException ("index", _("ArgRange_Array")); } if (count < 0 || count > (bytes.Length - index)) { throw new ArgumentOutOfRangeException ("count", _("ArgRange_Array")); } if (count == 0) return 0; fixed (byte *bptr = bytes) return InternalGetCharCount (bptr + index, count, leftOverBits, leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush); }
// This is internal and called by something else, internal override unsafe int GetCharCount(byte *bytes, int count, DecoderNLS decoder) { // Just assert, we're called internally so these should be safe, checked already Debug.Assert(bytes != null, "[ASCIIEncoding.GetCharCount]bytes is null"); Debug.Assert(count >= 0, "[ASCIIEncoding.GetCharCount]byteCount is negative"); // ASCII doesn't do best fit, so don't have to check for it, find out which decoder fallback we're using DecoderReplacementFallback fallback = null; if (decoder == null) { fallback = this.DecoderFallback as DecoderReplacementFallback; } else { fallback = decoder.Fallback as DecoderReplacementFallback; Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0, "[ASCIICodePageEncoding.GetCharCount]Expected empty fallback buffer"); } if (fallback != null && fallback.MaxCharCount == 1) { // Just return length, SBCS stay the same length because they don't map to surrogate // pairs and we don't have a decoder fallback. return(count); } // Only need decoder fallback buffer if not using default replacement fallback, no best fit for ASCII DecoderFallbackBuffer fallbackBuffer = null; // Have to do it the hard way. // Assume charCount will be == count int charCount = count; byte[] byteBuffer = new byte[1]; // Do it our fast way byte *byteEnd = bytes + count; // Quick loop while (bytes < byteEnd) { // Faster if don't use *bytes++; byte b = *bytes; bytes++; // If unknown we have to do fallback count if (b >= 0x80) { if (fallbackBuffer == null) { if (decoder == null) { fallbackBuffer = this.DecoderFallback.CreateFallbackBuffer(); } else { fallbackBuffer = decoder.FallbackBuffer; } fallbackBuffer.InternalInitialize(byteEnd - count, null); } // Use fallback buffer byteBuffer[0] = b; charCount--; // Have to unreserve the one we already allocated for b charCount += fallbackBuffer.InternalFallback(byteBuffer, bytes); } } // Fallback buffer must be empty Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0, "[ASCIIEncoding.GetCharCount]Expected Empty fallback buffer"); // Converted sequence is same length as input return(charCount); }
int GetChars (byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex, ref DecoderFallbackBuffer buffer) { if (bytes == null) throw new ArgumentNullException ("bytes"); if (chars == null) throw new ArgumentNullException ("chars"); if (byteIndex < 0 || byteIndex > bytes.Length) throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array")); if (byteCount < 0 || byteCount > (bytes.Length - byteIndex)) throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array")); if (charIndex < 0 || charIndex > chars.Length) throw new ArgumentOutOfRangeException ("charIndex", _("ArgRange_Array")); if ((chars.Length - charIndex) < byteCount) throw new ArgumentException (_("Arg_InsufficientSpace")); int count = byteCount; while (count-- > 0) { char c = (char) bytes [byteIndex++]; if (c < '\x80') chars [charIndex++] = c; else { if (buffer == null) buffer = DecoderFallback.CreateFallbackBuffer (); var thisByte = new byte[] { bytes [byteIndex-1] }; buffer.Fallback (thisByte, 0); while (buffer.Remaining > 0) { if (charIndex < chars.Length) { chars [charIndex++] = buffer.GetNextChar (); continue; } throw new ArgumentException ( "The output char buffer is too small to contain the " + "decoded characters."); } } } return byteCount; }
// This function is called when we want to flush the decoder state // (i.e. in case of invalid UTF-8 characters or interrupted sequences) internal unsafe static DecoderStatus InternalGetCharsFlush ( char* chars, int charCount, DecoderFallbackBuffer fallbackBuffer, DecoderStatus s, int bytesProcessed, ref int charsProcessed, ref uint leftBytes, ref uint leftBits, ref uint procBytes) { // if there is nothing to flush, then exit silently if(procBytes == 0) return DecoderStatus.Ok; // now we build a 'bytesUnknown' array with the // stored bytes in 'procBytes'. int extra = 0; for (uint t = procBytes; t != 0; extra++) t = t >> 8; byte [] bytesUnknown = new byte [extra]; for (int i = extra; i > 0; i--) bytesUnknown [i - 1] = (byte) ((procBytes >> (8 * (extra - i))) & 0xff); // partial reset: this condition avoids infinite loops if (s == DecoderStatus.InvalidSequence) leftBytes = 0; // call the fallback and cross fingers fallbackBuffer.Fallback (bytesUnknown, bytesProcessed - extra); if (chars != null) { while (fallbackBuffer.Remaining > 0) { if (charsProcessed >= charCount) return DecoderStatus.InsufficientSpace; chars [charsProcessed++] = fallbackBuffer.GetNextChar (); } } else charsProcessed += fallbackBuffer.Remaining; fallbackBuffer.Reset (); // recovery was succesful, flush decoder state leftBits = leftBytes = procBytes = 0; return DecoderStatus.Ok; }
[System.Security.SecurityCritical] // auto-generated public override unsafe int GetChars(byte* bytes, int byteCount, char* chars, int charCount, DecoderNLS decoder) { // Just need to ASSERT, this is called by something else internal that checked parameters already Debug.Assert(bytes != null, "[SBCSCodePageEncoding.GetChars]bytes is null"); Debug.Assert(byteCount >= 0, "[SBCSCodePageEncoding.GetChars]byteCount is negative"); Debug.Assert(chars != null, "[SBCSCodePageEncoding.GetChars]chars is null"); Debug.Assert(charCount >= 0, "[SBCSCodePageEncoding.GetChars]charCount is negative"); CheckMemorySection(); // See if we have best fit bool bUseBestFit = false; // Do it fast way if using ? replacement or best fit fallbacks byte* byteEnd = bytes + byteCount; byte* byteStart = bytes; char* charStart = chars; // Only need decoder fallback buffer if not using default replacement fallback or best fit fallback. DecoderReplacementFallback fallback = null; if (decoder == null) { fallback = DecoderFallback as DecoderReplacementFallback; bUseBestFit = DecoderFallback is InternalDecoderBestFitFallback; } else { fallback = decoder.Fallback as DecoderReplacementFallback; bUseBestFit = decoder.Fallback is InternalDecoderBestFitFallback; Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0, "[SBCSCodePageEncoding.GetChars]Expected empty fallback buffer at start"); } if (bUseBestFit || (fallback != null && fallback.MaxCharCount == 1)) { // Try it the fast way char replacementChar; if (fallback == null) replacementChar = '?'; // Best fit always has ? for fallback for SBCS else replacementChar = fallback.DefaultString[0]; // Need byteCount chars, otherwise too small buffer if (charCount < byteCount) { // Need at least 1 output byte, throw if must throw ThrowCharsOverflow(decoder, charCount < 1); // Not throwing, use what we can byteEnd = bytes + charCount; } // Quick loop, just do '?' replacement because we don't have fallbacks for decodings. while (bytes < byteEnd) { char c; if (bUseBestFit) { if (arrayBytesBestFit == null) { ReadBestFitTable(); } c = arrayBytesBestFit[*bytes]; } else c = _mapBytesToUnicode[*bytes]; bytes++; if (c == UNKNOWN_CHAR) // This is an invalid byte in the ASCII encoding. *chars = replacementChar; else *chars = c; chars++; } // bytes & chars used are the same if (decoder != null) decoder.m_bytesUsed = (int)(bytes - byteStart); return (int)(chars - charStart); } // Slower way's going to need a fallback buffer DecoderFallbackBuffer fallbackBuffer = null; byte[] byteBuffer = new byte[1]; char* charEnd = chars + charCount; DecoderFallbackBufferHelper fallbackHelper = new DecoderFallbackBufferHelper(decoder.FallbackBuffer); // Not quite so fast loop while (bytes < byteEnd) { // Faster if don't use *bytes++; char c = _mapBytesToUnicode[*bytes]; bytes++; // See if it was unknown if (c == UNKNOWN_CHAR) { // Make sure we have a fallback buffer if (fallbackBuffer == null) { if (decoder == null) fallbackBuffer = DecoderFallback.CreateFallbackBuffer(); else fallbackBuffer = decoder.FallbackBuffer; fallbackHelper = new DecoderFallbackBufferHelper(fallbackBuffer); fallbackHelper.InternalInitialize(byteEnd - byteCount, charEnd); } // Use fallback buffer Debug.Assert(bytes > byteStart, "[SBCSCodePageEncoding.GetChars]Expected bytes to have advanced already (unknown byte)"); byteBuffer[0] = *(bytes - 1); // Fallback adds fallback to chars, but doesn't increment chars unless the whole thing fits. if (!fallbackHelper.InternalFallback(byteBuffer, bytes, ref chars)) { // May or may not throw, but we didn't get this byte bytes--; // unused byte fallbackHelper.InternalReset(); // Didn't fall this back ThrowCharsOverflow(decoder, bytes == byteStart); // throw? break; // don't throw, but stop loop } } else { // Make sure we have buffer space if (chars >= charEnd) { Debug.Assert(bytes > byteStart, "[SBCSCodePageEncoding.GetChars]Expected bytes to have advanced already (known byte)"); bytes--; // unused byte ThrowCharsOverflow(decoder, bytes == byteStart); // throw? break; // don't throw, but stop loop } *(chars) = c; chars++; } } // Might have had decoder fallback stuff. if (decoder != null) decoder.m_bytesUsed = (int)(bytes - byteStart); // Expect Empty fallback buffer for GetChars Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0, "[SBCSEncoding.GetChars]Expected Empty fallback buffer at end"); return (int)(chars - charStart); }
[System.Security.SecurityCritical] // auto-generated public override unsafe int GetCharCount(byte* bytes, int count, DecoderNLS decoder) { // Just assert, we're called internally so these should be safe, checked already Debug.Assert(bytes != null, "[SBCSCodePageEncoding.GetCharCount]bytes is null"); Debug.Assert(count >= 0, "[SBCSCodePageEncoding.GetCharCount]byteCount is negative"); CheckMemorySection(); // See if we have best fit bool bUseBestFit = false; // Only need decoder fallback buffer if not using default replacement fallback or best fit fallback. DecoderReplacementFallback fallback = null; if (decoder == null) { fallback = DecoderFallback as DecoderReplacementFallback; bUseBestFit = DecoderFallback is InternalDecoderBestFitFallback; } else { fallback = decoder.Fallback as DecoderReplacementFallback; bUseBestFit = decoder.Fallback is InternalDecoderBestFitFallback; Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0, "[SBCSCodePageEncoding.GetChars]Expected empty fallback buffer at start"); } if (bUseBestFit || (fallback != null && fallback.MaxCharCount == 1)) { // Just return length, SBCS stay the same length because they don't map to surrogate // pairs and we don't have a decoder fallback. return count; } // Might need one of these later DecoderFallbackBuffer fallbackBuffer = null; DecoderFallbackBufferHelper fallbackHelper = new DecoderFallbackBufferHelper(fallbackBuffer); // Have to do it the hard way. // Assume charCount will be == count int charCount = count; byte[] byteBuffer = new byte[1]; // Do it our fast way byte* byteEnd = bytes + count; // Quick loop while (bytes < byteEnd) { // Faster if don't use *bytes++; char c; c = _mapBytesToUnicode[*bytes]; bytes++; // If unknown we have to do fallback count if (c == UNKNOWN_CHAR) { // Must have a fallback buffer if (fallbackBuffer == null) { // Need to adjust count so we get real start if (decoder == null) fallbackBuffer = DecoderFallback.CreateFallbackBuffer(); else fallbackBuffer = decoder.FallbackBuffer; fallbackHelper = new DecoderFallbackBufferHelper(fallbackBuffer); fallbackHelper.InternalInitialize(byteEnd - count, null); } // Use fallback buffer byteBuffer[0] = *(bytes - 1); charCount--; // We'd already reserved one for *(bytes-1) charCount += fallbackHelper.InternalFallback(byteBuffer, bytes); } } // Fallback buffer must be empty Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0, "[SBCSEncoding.GetCharCount]Expected Empty fallback buffer at end"); // Converted sequence is same length as input return charCount; }
internal override unsafe int GetChars(byte *bytes, int byteCount, char *chars, int charCount, DecoderNLS decoder) { // Just need to ASSERT, this is called by something else internal that checked parameters already Debug.Assert(bytes != null, "[ASCIIEncoding.GetChars]bytes is null"); Debug.Assert(byteCount >= 0, "[ASCIIEncoding.GetChars]byteCount is negative"); Debug.Assert(chars != null, "[ASCIIEncoding.GetChars]chars is null"); Debug.Assert(charCount >= 0, "[ASCIIEncoding.GetChars]charCount is negative"); // Do it fast way if using ? replacement fallback byte *byteEnd = bytes + byteCount; byte *byteStart = bytes; char *charStart = chars; // Note: ASCII doesn't do best fit, but we have to fallback if they use something > 0x7f // Only need decoder fallback buffer if not using ? fallback. // ASCII doesn't do best fit, so don't have to check for it, find out which decoder fallback we're using DecoderReplacementFallback fallback = null; if (decoder == null) { fallback = this.DecoderFallback as DecoderReplacementFallback; } else { fallback = decoder.Fallback as DecoderReplacementFallback; Debug.Assert(!decoder.m_throwOnOverflow || !decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0, "[ASCIICodePageEncoding.GetChars]Expected empty fallback buffer"); } if (fallback != null && fallback.MaxCharCount == 1) { // Try it the fast way char replacementChar = fallback.DefaultString[0]; // Need byteCount chars, otherwise too small buffer if (charCount < byteCount) { // Need at least 1 output byte, throw if must throw ThrowCharsOverflow(decoder, charCount < 1); // Not throwing, use what we can byteEnd = bytes + charCount; } // Quick loop, just do '?' replacement because we don't have fallbacks for decodings. while (bytes < byteEnd) { byte b = *(bytes++); if (b >= 0x80) { // This is an invalid byte in the ASCII encoding. *(chars++) = replacementChar; } else { *(chars++) = unchecked ((char)b); } } // bytes & chars used are the same if (decoder != null) { decoder.m_bytesUsed = (int)(bytes - byteStart); } return((int)(chars - charStart)); } // Slower way's going to need a fallback buffer DecoderFallbackBuffer fallbackBuffer = null; byte[] byteBuffer = new byte[1]; char * charEnd = chars + charCount; // Not quite so fast loop while (bytes < byteEnd) { // Faster if don't use *bytes++; byte b = *(bytes); bytes++; if (b >= 0x80) { // This is an invalid byte in the ASCII encoding. if (fallbackBuffer == null) { if (decoder == null) { fallbackBuffer = this.DecoderFallback.CreateFallbackBuffer(); } else { fallbackBuffer = decoder.FallbackBuffer; } fallbackBuffer.InternalInitialize(byteEnd - byteCount, charEnd); } // Use fallback buffer byteBuffer[0] = b; // Note that chars won't get updated unless this succeeds if (!fallbackBuffer.InternalFallback(byteBuffer, bytes, ref chars)) { // May or may not throw, but we didn't get this byte Debug.Assert(bytes > byteStart || chars == charStart, "[ASCIIEncoding.GetChars]Expected bytes to have advanced already (fallback case)"); bytes--; // unused byte fallbackBuffer.InternalReset(); // Didn't fall this back ThrowCharsOverflow(decoder, chars == charStart); // throw? break; // don't throw, but stop loop } } else { // Make sure we have buffer space if (chars >= charEnd) { Debug.Assert(bytes > byteStart || chars == charStart, "[ASCIIEncoding.GetChars]Expected bytes to have advanced already (normal case)"); bytes--; // unused byte ThrowCharsOverflow(decoder, chars == charStart); // throw? break; // don't throw, but stop loop } *(chars) = unchecked ((char)b); chars++; } } // Might have had decoder fallback stuff. if (decoder != null) { decoder.m_bytesUsed = (int)(bytes - byteStart); } // Expect Empty fallback buffer for GetChars Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0, "[ASCIIEncoding.GetChars]Expected Empty fallback buffer"); return((int)(chars - charStart)); }
private unsafe int FallbackInvalidByteSequence(byte* pSrc, int ch, DecoderFallbackBuffer fallback) { byte[] bytesUnknown = this.GetBytesUnknown(ref pSrc, ch); return fallback.InternalFallback(bytesUnknown, pSrc); }
// for GetCharCount() static unsafe int Fallback (object provider, ref DecoderFallbackBuffer buffer, ref byte [] bufferArg, byte* bytes, long index, uint size) { if (buffer == null) { DecoderFallback fb = provider as DecoderFallback; if (fb != null) buffer = fb.CreateFallbackBuffer (); else buffer = ((Decoder) provider).FallbackBuffer; } if (bufferArg == null) bufferArg = new byte [1]; int ret = 0; for (int i = 0; i < size; i++) { bufferArg [0] = bytes [(int) index + i]; buffer.Fallback (bufferArg, 0); ret += buffer.Remaining; buffer.Reset (); } return ret; }
// InternalGetChars processor. Can decode or count space needed for // decoding, depending on the enabled mode: // - decoder // enabled when charCount >= 0 (but chars may be null) // - counter // enabled when chars == null && charCount < 0 internal unsafe static DecoderStatus InternalGetChars ( byte* bytes, int byteCount, char* chars, int charCount, DecoderFallbackBuffer fallbackBuffer, out int bytesProcessed, out int charsProcessed, ref uint leftBytes, ref uint leftBits, ref uint procBytes, bool flush) { DecoderStatus s; int t_bytesProcessed, t_charsProcessed; // Validate parameters if (byteCount < 0) throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_NonNegative")); else if (byteCount > 0 && bytes == null) throw new ArgumentNullException ("bytes"); if (chars == null) { if (charCount > 0) throw new ArgumentNullException ("chars"); } else { if (charCount < 0) throw new ArgumentOutOfRangeException ("charCount", _("ArgRange_NonNegative")); } // reset counters charsProcessed = 0; bytesProcessed = 0; // byte processing loop while (byteCount - bytesProcessed > 0) { // fetch a char from the input byte array s = chars != null ? InternalGetChar ( bytes + bytesProcessed, byteCount - bytesProcessed, chars + charsProcessed, charCount - charsProcessed, out t_bytesProcessed, out t_charsProcessed, ref leftBytes, ref leftBits, ref procBytes) : InternalGetChar ( bytes + bytesProcessed, byteCount - bytesProcessed, null, charCount, out t_bytesProcessed, out t_charsProcessed, ref leftBytes, ref leftBits, ref procBytes); // if not enough space return here // NOTE: maybe we should restore the original encoder // state ... we should check what ms do in this case if(s == DecoderStatus.InsufficientSpace) return DecoderStatus.InsufficientSpace; // update counters charsProcessed += t_charsProcessed; bytesProcessed += t_bytesProcessed; switch (s) { case DecoderStatus.Ok: break; // everything OK :D case DecoderStatus.Overlong: case DecoderStatus.InvalidSequence: case DecoderStatus.InvalidStart: case DecoderStatus.InvalidChar: case DecoderStatus.SurrogateFound: s = InternalGetCharsFlush ( chars, charCount, fallbackBuffer, s, bytesProcessed, ref charsProcessed, ref leftBytes, ref leftBits, ref procBytes); if (s != DecoderStatus.Ok) return s; break; case DecoderStatus.InputRunOut: return flush ? InternalGetCharsFlush ( chars, charCount, fallbackBuffer, s, bytesProcessed, ref charsProcessed, ref leftBytes, ref leftBits, ref procBytes) : DecoderStatus.InputRunOut; } } return flush ? InternalGetCharsFlush ( chars, charCount, fallbackBuffer, DecoderStatus.Ok, bytesProcessed, ref charsProcessed, ref leftBytes, ref leftBits, ref procBytes) : DecoderStatus.Ok; }
private unsafe static int InternalGetCharCount ( byte* bytes, int count, uint leftOverBits, uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush) { int index = 0; int length = 0; if (leftOverCount == 0) { int end = index + count; for (; index < end; index++, count--) { if (bytes [index] < 0x80) length++; else break; } } // Determine the number of characters that we have. uint ch; uint leftBits = leftOverBits; uint leftSoFar = (leftOverCount & (uint)0x0F); uint leftSize = ((leftOverCount >> 4) & (uint)0x0F); while (count > 0) { ch = (uint)(bytes[index++]); --count; if (leftSize == 0) { // Process a UTF-8 start character. if (ch < (uint)0x0080) { // Single-byte UTF-8 character. ++length; } else if ((ch & (uint)0xE0) == (uint)0xC0) { // Double-byte UTF-8 character. leftBits = (ch & (uint)0x1F); leftSoFar = 1; leftSize = 2; } else if ((ch & (uint)0xF0) == (uint)0xE0) { // Three-byte UTF-8 character. leftBits = (ch & (uint)0x0F); leftSoFar = 1; leftSize = 3; } else if ((ch & (uint)0xF8) == (uint)0xF0) { // Four-byte UTF-8 character. leftBits = (ch & (uint)0x07); leftSoFar = 1; leftSize = 4; } else if ((ch & (uint)0xFC) == (uint)0xF8) { // Five-byte UTF-8 character. leftBits = (ch & (uint)0x03); leftSoFar = 1; leftSize = 5; } else if ((ch & (uint)0xFE) == (uint)0xFC) { // Six-byte UTF-8 character. leftBits = (ch & (uint)0x03); leftSoFar = 1; leftSize = 6; } else { // Invalid UTF-8 start character. length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - 1, 1); } } else { // Process an extra byte in a multi-byte sequence. if ((ch & (uint)0xC0) == (uint)0x80) { leftBits = ((leftBits << 6) | (ch & (uint)0x3F)); if (++leftSoFar >= leftSize) { // We have a complete character now. if (leftBits < (uint)0x10000) { // is it an overlong ? bool overlong = false; switch (leftSize) { case 2: overlong = (leftBits <= 0x7F); break; case 3: overlong = (leftBits <= 0x07FF); break; case 4: overlong = (leftBits <= 0xFFFF); break; case 5: overlong = (leftBits <= 0x1FFFFF); break; case 6: overlong = (leftBits <= 0x03FFFFFF); break; } if (overlong) { length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar); } else if ((leftBits & 0xF800) == 0xD800) { // UTF-8 doesn't use surrogate characters length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar); } else ++length; } else if (leftBits < (uint)0x110000) { length += 2; } else { length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar); } leftSize = 0; } } else { // Invalid UTF-8 sequence: clear and restart. length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar); leftSize = 0; --index; ++count; } } } if (flush && leftSize != 0) { // We had left-over bytes that didn't make up // a complete UTF-8 character sequence. length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar); } // Return the final length to the caller. return length; }
internal unsafe static DecoderStatus InternalGetCharsCount ( byte* bytes, int byteCount, DecoderFallbackBuffer fallbackBuffer, out int bytesProcessed, out int charsProcessed, ref uint leftBytes, ref uint leftBits, ref uint procBytes, bool flush) { if (byteCount < 0) throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array")); return InternalGetChars ( bytes, byteCount, null, -1, fallbackBuffer, out bytesProcessed, out charsProcessed, ref leftBytes, ref leftBits, ref procBytes, flush); }
// for GetChars() static unsafe void Fallback (object provider, ref DecoderFallbackBuffer buffer, ref byte [] bufferArg, byte* bytes, long byteIndex, uint size, char* chars, ref int charIndex) { if (buffer == null) { DecoderFallback fb = provider as DecoderFallback; if (fb != null) buffer = fb.CreateFallbackBuffer (); else buffer = ((Decoder) provider).FallbackBuffer; } if (bufferArg == null) bufferArg = new byte [1]; for (int i = 0; i < size; i++) { bufferArg [0] = bytes [byteIndex + i]; buffer.Fallback (bufferArg, 0); while (buffer.Remaining > 0) chars [charIndex++] = buffer.GetNextChar (); buffer.Reset (); } }
internal unsafe static DecoderStatus InternalGetCharsCount ( byte[] bytes, int byteIndex, int byteCount, DecoderFallbackBuffer fallbackBuffer, out int bytesProcessed, out int charsProcessed, ref uint leftBytes, ref uint leftBits, ref uint procBytes, bool flush) { if (bytes == null) throw new ArgumentNullException ("bytes"); if (byteIndex < 0 || byteIndex > bytes.Length) throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array")); if (byteCount < 0 || byteCount > (bytes.Length - byteIndex)) throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_Array")); fixed (byte* bptr = bytes) { return InternalGetChars ( bptr + byteIndex, byteCount, null, -1, fallbackBuffer, out bytesProcessed, out charsProcessed, ref leftBytes, ref leftBits, ref procBytes, flush); } }
private unsafe static int InternalGetChars ( byte* bytes, int byteCount, char* chars, int charCount, ref uint leftOverBits, ref uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush) { int charIndex = 0, byteIndex = 0; int length = charCount; int posn = charIndex; if (leftOverCount == 0) { int end = byteIndex + byteCount; for (; byteIndex < end; posn++, byteIndex++, byteCount--) { if (bytes [byteIndex] < 0x80) chars [posn] = (char) bytes [byteIndex]; else break; } } // Convert the bytes into the output buffer. uint ch; uint leftBits = leftOverBits; uint leftSoFar = (leftOverCount & (uint)0x0F); uint leftSize = ((leftOverCount >> 4) & (uint)0x0F); int byteEnd = byteIndex + byteCount; for(; byteIndex < byteEnd; byteIndex++) { // Fetch the next character from the byte buffer. ch = (uint)(bytes[byteIndex]); if (leftSize == 0) { // Process a UTF-8 start character. if (ch < (uint)0x0080) { // Single-byte UTF-8 character. if (posn >= length) { throw new ArgumentException (_("Arg_InsufficientSpace"), "chars"); } chars[posn++] = (char)ch; } else if ((ch & (uint)0xE0) == (uint)0xC0) { // Double-byte UTF-8 character. leftBits = (ch & (uint)0x1F); leftSoFar = 1; leftSize = 2; } else if ((ch & (uint)0xF0) == (uint)0xE0) { // Three-byte UTF-8 character. leftBits = (ch & (uint)0x0F); leftSoFar = 1; leftSize = 3; } else if ((ch & (uint)0xF8) == (uint)0xF0) { // Four-byte UTF-8 character. leftBits = (ch & (uint)0x07); leftSoFar = 1; leftSize = 4; } else if ((ch & (uint)0xFC) == (uint)0xF8) { // Five-byte UTF-8 character. leftBits = (ch & (uint)0x03); leftSoFar = 1; leftSize = 5; } else if ((ch & (uint)0xFE) == (uint)0xFC) { // Six-byte UTF-8 character. leftBits = (ch & (uint)0x03); leftSoFar = 1; leftSize = 6; } else { // Invalid UTF-8 start character. Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex, 1, chars, ref posn); } } else { // Process an extra byte in a multi-byte sequence. if ((ch & (uint)0xC0) == (uint)0x80) { leftBits = ((leftBits << 6) | (ch & (uint)0x3F)); if (++leftSoFar >= leftSize) { // We have a complete character now. if (leftBits < (uint)0x10000) { // is it an overlong ? bool overlong = false; switch (leftSize) { case 2: overlong = (leftBits <= 0x7F); break; case 3: overlong = (leftBits <= 0x07FF); break; case 4: overlong = (leftBits <= 0xFFFF); break; case 5: overlong = (leftBits <= 0x1FFFFF); break; case 6: overlong = (leftBits <= 0x03FFFFFF); break; } if (overlong) { Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn); } else if ((leftBits & 0xF800) == 0xD800) { // UTF-8 doesn't use surrogate characters Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn); } else { if (posn >= length) { throw new ArgumentException (_("Arg_InsufficientSpace"), "chars"); } chars[posn++] = (char)leftBits; } } else if (leftBits < (uint)0x110000) { if ((posn + 2) > length) { throw new ArgumentException (_("Arg_InsufficientSpace"), "chars"); } leftBits -= (uint)0x10000; chars[posn++] = (char)((leftBits >> 10) + (uint)0xD800); chars[posn++] = (char)((leftBits & (uint)0x3FF) + (uint)0xDC00); } else { Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn); } leftSize = 0; } } else { // Invalid UTF-8 sequence: clear and restart. Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn); leftSize = 0; --byteIndex; } } } if (flush && leftSize != 0) { // We had left-over bytes that didn't make up // a complete UTF-8 character sequence. Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn); } leftOverBits = leftBits; leftOverCount = (leftSoFar | (leftSize << 4)); // Return the final length to the caller. return posn - charIndex; }
// Internal version of "GetCharCount" which can handle a rolling // state between multiple calls to this method. #if NET_2_0 private unsafe static int InternalGetCharCount ( byte[] bytes, int index, int count, uint leftOverBits, uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
internal unsafe EncodingCharBuffer(Encoding enc, DecoderNLS decoder, char* charStart, int charCount, byte* byteStart, int byteCount) { this.enc = enc; this.decoder = decoder; this.chars = charStart; this.charStart = charStart; this.charEnd = charStart + charCount; this.byteStart = byteStart; this.bytes = byteStart; this.byteEnd = byteStart + byteCount; if (this.decoder == null) { this.fallbackBuffer = enc.DecoderFallback.CreateFallbackBuffer(); } else { this.fallbackBuffer = this.decoder.FallbackBuffer; } this.fallbackBuffer.InternalInitialize(this.bytes, this.charEnd); }
private unsafe static int InternalGetChars ( byte* bytes, int byteCount, char* chars, int charCount, ref uint leftOverBits, ref uint leftOverCount, object provider, ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
[System.Security.SecurityCritical] // auto-generated private unsafe int FallbackInvalidByteSequence( byte* pSrc, int ch, DecoderFallbackBuffer fallback) { // Get our byte[] byte[] bytesUnknown = GetBytesUnknown(ref pSrc, ch); // Do the actual fallback int count = fallback.InternalFallback(bytesUnknown, pSrc); // # of fallback chars expected. // Note that we only get here for "long" sequences, and have already unreserved // the count that we prereserved for the input bytes return count; }
int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex, ref DecoderFallbackBuffer buffer) { if (bytes == null) { throw new ArgumentNullException("bytes"); } if (chars == null) { throw new ArgumentNullException("chars"); } if (byteIndex < 0 || byteIndex > bytes.Length) { throw new ArgumentOutOfRangeException("byteIndex", _("ArgRange_Array")); } if (byteCount < 0 || byteCount > (bytes.Length - byteIndex)) { throw new ArgumentOutOfRangeException("byteCount", _("ArgRange_Array")); } if (charIndex < 0 || charIndex > chars.Length) { throw new ArgumentOutOfRangeException("charIndex", _("ArgRange_Array")); } if ((chars.Length - charIndex) < byteCount) { throw new ArgumentException(_("Arg_InsufficientSpace")); } int count = byteCount; while (count-- > 0) { char c = (char)bytes [byteIndex++]; if (c < '\x80') { chars [charIndex++] = c; } else { if (buffer == null) { buffer = DecoderFallback.CreateFallbackBuffer(); } var thisByte = new byte[] { bytes [byteIndex - 1] }; buffer.Fallback(thisByte, 0); while (buffer.Remaining > 0) { if (charIndex < chars.Length) { chars [charIndex++] = buffer.GetNextChar(); continue; } throw new ArgumentException( "The output char buffer is too small to contain the " + "decoded characters."); } } } return(byteCount); }