public void HandleFallback(ref EncoderFallbackBuffer buffer, char[] chars, ref int charIndex, ref int charCount, byte[] bytes, ref int byteIndex, ref int byteCount, object state) { if (buffer == null) buffer = EncoderFallback.CreateFallbackBuffer(); if (charCount > 1 && (Char.IsSurrogate(chars[charIndex]) && Char.IsSurrogate(chars[charIndex + 1]))) { buffer.Fallback (chars[charIndex], chars[charIndex + 1], charIndex); charIndex++; charCount--; } else buffer.Fallback (chars[charIndex], charIndex); char[] tmp = new char[buffer.Remaining]; int idx = 0; while (buffer.Remaining > 0) tmp[idx++] = buffer.GetNextChar(); var len = state == null ? GetBytes(tmp, 0, tmp.Length, bytes, byteIndex) : GetBytesInternal(tmp, 0, tmp.Length, bytes, byteIndex, true, state); byteIndex += len; byteCount -= len; }
private int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex, EncoderFallbackBuffer efb) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; if (c > 0x7f) { if (efb == null) { efb = EncoderFallback.CreateFallbackBuffer(); } if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { bytes[byteIndex++] = (byte)efb.GetNextChar(); outputBytes++; } } } else { bytes[byteIndex++] = (byte)c; outputBytes++; } charIndex++; } return(outputBytes); }
private int GetByteCount(char[] chars, int index, int count, EncoderFallbackBuffer efb) { int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; if (c > 0x7f) { if (efb == null) { efb = EncoderFallback.CreateFallbackBuffer(); } if (efb.Fallback(c, index)) { byteCount += efb.Remaining; while (efb.GetNextChar() != char.MinValue) /* empty */ } { } } else { byteCount++; } index++; } return(byteCount); }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; #if FEATURE_ENCODING if (c > 0x7f) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { bytes[byteIndex++] = (byte)efb.GetNextChar(); outputBytes++; } } } else { bytes[byteIndex++] = (byte)c; outputBytes++; } #else bytes[byteIndex++] = (byte)c; outputBytes++; #endif charIndex++; } return(outputBytes); }
public override int GetByteCount(char[] chars, int index, int count) { #if FEATURE_ENCODING int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; if (c > 0x7f) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, index)) { byteCount += efb.Remaining; } } else { byteCount++; } index++; } return(byteCount); #else return(count); #endif }
public override bool Fallback(char charUnknownHigh, char charUnknownLow, int index) { // check input surrogate pair if (!char.IsSurrogatePair(charUnknownHigh, charUnknownLow)) { throw XmlConvert.CreateInvalidSurrogatePairException(charUnknownHigh, charUnknownLow); } // If we are already in fallback, throw, it's probably at the suspect character in charEntity if (_charEntityIndex >= 0) { (new EncoderExceptionFallback()).CreateFallbackBuffer().Fallback(charUnknownHigh, charUnknownLow, index); } if (_parent.CanReplaceAt(index)) { // Create the replacement character entity _charEntity = string.Format(CultureInfo.InvariantCulture, "&#x{0:X};", new object[] { SurrogateCharToUtf32(charUnknownHigh, charUnknownLow) }); _charEntityIndex = 0; return(true); } else { EncoderFallbackBuffer errorFallbackBuffer = (new EncoderExceptionFallback()).CreateFallbackBuffer(); errorFallbackBuffer.Fallback(charUnknownHigh, charUnknownLow, index); return(false); } }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; char val; if (!_map.Mapping.TryGetValue((int)c, out val)) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { bytes[byteIndex++] = (byte)_map.Mapping[(int)efb.GetNextChar()]; outputBytes++; } } } else { bytes[byteIndex++] = (byte)val; outputBytes++; } charIndex++; } return(outputBytes); }
public override int GetByteCount(char[] chars, int index, int count) { int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; if (!_map.Mapping.TryGetValue(c, out _)) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, index)) { byteCount += efb.Remaining; } } else { byteCount++; } index++; } return(byteCount); }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; object val; object obj = (int)c; if (!_map.TryGetValue(obj, out val) || (val == null && _errors == "strict")) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { obj = (int)efb.GetNextChar(); bytes[byteIndex++] = (byte)((int)_map[obj]); outputBytes++; } } } else if (val == null) { throw PythonOps.UnicodeEncodeError("charmap", c.ToString(), charIndex, charIndex + 1, "character maps to <undefined>"); } else if (val is string) { string v = val as string; for (int i = 0; i < v.Length; i++) { bytes[byteIndex++] = (byte)v[i]; outputBytes++; } } else if (val is int) { bytes[byteIndex++] = (byte)(int)val; outputBytes++; } else { throw PythonOps.TypeError("charmap must be an int, str, or None"); } charIndex++; } return(outputBytes); }
public unsafe void HandleFallback(ref EncoderFallbackBuffer buffer, char *chars, ref int charIndex, ref int charCount, byte *bytes, ref int byteIndex, ref int byteCount, object state) { if (buffer == null) { buffer = EncoderFallback.CreateFallbackBuffer(); } if (charCount > 1 && (Char.IsSurrogate(chars [charIndex]) && Char.IsSurrogate(chars [charIndex + 1]))) { buffer.Fallback(chars [charIndex], chars [charIndex + 1], charIndex); charIndex++; charCount--; } else { buffer.Fallback(chars [charIndex], charIndex); } char [] tmp = new char [buffer.Remaining]; int idx = 0; while (buffer.Remaining > 0) tmp [idx++] = buffer.GetNextChar(); fixed(char *tmparr = tmp) { var outbytes = bytes == null ? null : bytes + byteIndex; var len = state == null? GetBytes(tmparr, tmp.Length, outbytes, byteCount) : GetBytesInternal(tmparr, tmp.Length, outbytes, byteCount, true, state); byteIndex += len; byteCount -= len; } }
public void HandleFallback(ref EncoderFallbackBuffer buffer, char[] chars, ref int charIndex, ref int charCount, byte[] bytes, ref int byteIndex, ref int byteCount, object state) { if (buffer == null) { buffer = EncoderFallback.CreateFallbackBuffer(); } // THIS IS WERE THE BUG IS!! (pruiz) if (charCount > 1 && (Char.IsSurrogate(chars[charIndex]) && Char.IsSurrogate(chars[charIndex + 1]))) { buffer.Fallback(chars[charIndex], chars[charIndex + 1], charIndex); charIndex++; charCount--; } else { buffer.Fallback(chars[charIndex], charIndex); } char[] tmp = new char[buffer.Remaining]; int idx = 0; while (buffer.Remaining > 0) { tmp[idx++] = buffer.GetNextChar(); } var len = state == null? GetBytes(tmp, 0, tmp.Length, bytes, byteIndex) : GetBytesInternal(tmp, 0, tmp.Length, bytes, byteIndex, true, state); byteIndex += len; byteCount -= len; }
public override int GetByteCount(char[] chars, int index, int count) { int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; object val; object charObj = (int)c; if (!_map.TryGetValue(charObj, out val) || (val == null && _errors == "strict")) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, index)) { byteCount += efb.Remaining; } } else if (val == null) { throw PythonOps.UnicodeEncodeError("charmap", c.ToString(), index, index + 1, "character maps to <undefined>"); } else if (val is string) { byteCount += ((string)val).Length; } else if (val is int) { byteCount++; } else { throw PythonOps.TypeError("charmap must be an int, str, or None"); } index++; } return(byteCount); }
public static void EncoderReplacementFallbackBufferTest() { Encoder encoder = Encoding.GetEncoding("us-ascii", new EncoderReplacementFallback(), new DecoderReplacementFallback()).GetEncoder(); char [] chars = new char [] { '\uFFFF' }; byte [] bytes = new byte [2]; EncoderFallbackBuffer fallbackBuffer = encoder.FallbackBuffer; Assert.True(fallbackBuffer is EncoderReplacementFallbackBuffer, "Expected to get EncoderReplacementFallbackBuffer type"); Assert.True(fallbackBuffer.Fallback(chars[0], 0), "Expected we fallback on the given buffer"); Assert.Equal(1, fallbackBuffer.Remaining); Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are at the Beginning of the buffer"); Assert.Equal('?', fallbackBuffer.GetNextChar()); Assert.True(fallbackBuffer.MovePrevious(), "Expected we can move back on the replacement buffer"); Assert.Equal('?', fallbackBuffer.GetNextChar()); fallbackBuffer.Reset(); Assert.Equal(0, fallbackBuffer.Remaining); Assert.Equal('\u0000', fallbackBuffer.GetNextChar()); Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are rest the buffer"); }
public override bool Fallback(char charUnknown, int index) { // If we are already in fallback, throw, it's probably at the suspect character in charEntity if (_charEntityIndex >= 0) { (new EncoderExceptionFallback()).CreateFallbackBuffer().Fallback(charUnknown, index); } // find out if we can replace the character with entity if (_parent.CanReplaceAt(index)) { // Create the replacement character entity _charEntity = string.Create(null, stackalloc char[64], $"&#x{(int)charUnknown:X};"); _charEntityIndex = 0; return(true); } EncoderFallbackBuffer errorFallbackBuffer = (new EncoderExceptionFallback()).CreateFallbackBuffer(); errorFallbackBuffer.Fallback(charUnknown, index); return(false); }
public override bool Fallback(char charUnknown, int index) { // If we are already in fallback, throw, it's probably at the suspect character in charEntity if (_charEntityIndex >= 0) { (new EncoderExceptionFallback()).CreateFallbackBuffer().Fallback(charUnknown, index); } // find out if we can replace the character with entity if (_parent.CanReplaceAt(index)) { // Create the replacement character entity _charEntity = string.Format(CultureInfo.InvariantCulture, "&#x{0:X};", new object[] { (int)charUnknown }); _charEntityIndex = 0; return(true); } else { EncoderFallbackBuffer errorFallbackBuffer = (new EncoderExceptionFallback()).CreateFallbackBuffer(); errorFallbackBuffer.Fallback(charUnknown, index); return(false); } }
private int GetBytesInternal(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { var outpos = byteIndex; for (var inpos = charIndex; inpos < (charIndex + charCount); inpos++) { var character = chars[inpos]; var @byte = NOCHAR; var escape = false; if (character < Ucs2ToGsm.Length) { @byte = Ucs2ToGsm[character]; if (@byte == ESCAPE) { escape = true; @byte = Ucs2ToGsmExtended[character]; } } else if (character >= Ucs2GclToGsmBase && character <= Ucs2GclToGsmMax) { escape = true; @byte = Ucs2GclToGsm[character - Ucs2GclToGsmBase]; } else if (character == '\x20AC') // Euro sign. { escape = true; @byte = 0x65; } if (@byte == NOCHAR) { char tmp; EncoderFallbackBuffer.Fallback(character, inpos); while ((tmp = EncoderFallbackBuffer.GetNextChar()) != 0) { if (bytes != null) { bytes[outpos++] = Ucs2ToGsm[tmp]; // FIXME: Character might not be a 7-bit one.. } else { outpos++; } } EncoderFallbackBuffer.Reset(); } else { if (bytes != null) { if (escape) { bytes[outpos++] = ESCAPE; } bytes[outpos++] = @byte; } else { outpos += escape ? 2 : 1; } } } return(outpos - byteIndex); }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex, bool flush) { if (chars == null) { throw new ArgumentNullException("chars"); } if (charIndex < 0) { throw new ArgumentOutOfRangeException("charIndex"); } if (charCount < 0) { throw new ArgumentOutOfRangeException("charCount"); } if (charIndex + charCount > chars.Length) { throw new ArgumentOutOfRangeException("chars"); } if (bytes == null) { throw new ArgumentNullException("bytes"); } if (byteIndex < 0 || byteIndex >= bytes.Length) { throw new ArgumentOutOfRangeException("byteIndex"); } // The fallbackBuffer is created on-demand. The instance // FallbackBuffer isn't used because it wouldn't be thread safe. EncoderFallbackBuffer fallbackBuffer = null; // Written only on flush = false char?buffer = Buffer; int charCount2 = charIndex + charCount; int byteIndex2 = byteIndex; for (; charIndex < charCount2; charIndex++) { char ch = chars[charIndex]; if (buffer != null) { // If we have a High/Low surrogates couple, we pass them // together if (char.IsLowSurrogate(ch)) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } if (fallbackBuffer.Fallback(buffer.Value, ch, charIndex - 1)) { HandleFallback(fallbackBuffer, bytes, ref byteIndex2); } buffer = null; continue; } else { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } // First we pass the High surrogate to the Fallback if (fallbackBuffer.Fallback(buffer.Value, charIndex - 1)) { HandleFallback(fallbackBuffer, bytes, ref byteIndex2); } buffer = null; // Then we fall-through normal handling } } // High/low surrogate handling, done through buffer if (char.IsHighSurrogate(ch)) { buffer = ch; } else { byte b; if (!CharToByte(ch, out b)) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } // Fallback if (fallbackBuffer.Fallback(ch, charIndex)) { HandleFallback(fallbackBuffer, bytes, ref byteIndex2); } continue; } // Recognized character WriteByte(bytes, ref byteIndex2, b); } } if (flush) { if (buffer != null) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } if (fallbackBuffer.Fallback(buffer.Value, charIndex - 1)) { HandleFallback(fallbackBuffer, bytes, ref byteIndex2); } } } else { Buffer = buffer; } return(byteIndex2 - byteIndex); }
public override int GetByteCount(char[] chars, int index, int count, bool flush) { if (chars == null) { throw new ArgumentNullException("chars"); } if (index < 0) { throw new ArgumentOutOfRangeException("index"); } if (count < 0) { throw new ArgumentOutOfRangeException("count"); } if (index + count > chars.Length) { throw new ArgumentOutOfRangeException("chars"); } // The fallbackBuffer is created on-demand. The instance // FallbackBuffer isn't used because it wouldn't be thread safe. EncoderFallbackBuffer fallbackBuffer = null; // Written only on flush = false char?buffer = Buffer; int ret = 0; int count2 = index + count; for (; index < count2; index++) { char ch = chars[index]; if (buffer != null) { // If we have a High/Low surrogates couple, we pass them // together if (char.IsLowSurrogate(ch)) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } if (fallbackBuffer.Fallback(buffer.Value, ch, index - 1)) { while (fallbackBuffer.Remaining > 0) { fallbackBuffer.GetNextChar(); ret++; } } buffer = null; continue; } else { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } // First we pass the High surrogate to the Fallback if (fallbackBuffer.Fallback(buffer.Value, index - 1)) { while (fallbackBuffer.Remaining > 0) { fallbackBuffer.GetNextChar(); ret++; } } buffer = null; // Then we fall-through normal handling } } // High/low surrogate handling, done through buffer if (char.IsHighSurrogate(ch)) { buffer = ch; } else { byte b; if (!CharToByte(ch, out b)) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } // Fallback if (fallbackBuffer.Fallback(ch, index)) { while (fallbackBuffer.Remaining > 0) { fallbackBuffer.GetNextChar(); ret++; } } continue; } ret++; } } if (flush) { if (buffer != null) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } if (fallbackBuffer.Fallback(buffer.Value, index - 1)) { while (fallbackBuffer.Remaining > 0) { fallbackBuffer.GetNextChar(); ret++; } } } } else { Buffer = buffer; } return(ret); }
int InternalGetBytes(IEnumerable<char> chars, int charLength, int charIndex, int charCount, byte[] bytes, int byteIndex, ref EncoderFallbackBuffer buffer, ref char[] fallback_chars) { if (bytes == null) throw new ArgumentNullException("bytes"); if (charIndex < 0 || charIndex > charLength) throw new ArgumentOutOfRangeException("charIndex"); if (charCount < 0 || charCount > (charLength - charIndex)) throw new ArgumentOutOfRangeException("charCount"); if (byteIndex < 0 || byteIndex > bytes.Length) throw new ArgumentOutOfRangeException("byteIndex"); if ((bytes.Length - byteIndex) < charCount) throw new ArgumentException("InsufficientSpace"); using (var charIter = chars.GetEnumerator()) { for (var i = 0; i <= charIndex; ++i) charIter.MoveNext(); var count = charCount; while (count-- > 0) { var ch = charIter.Current; ++charIndex; charIter.MoveNext(); if (ch < (char) 0x80) { bytes[byteIndex++] = (byte) ch; } else { if (buffer == null) buffer = EncoderFallback.CreateFallbackBuffer(); if (Char.IsSurrogate(ch) && count > 1 && Char.IsSurrogate(charIter.Current)) buffer.Fallback(ch, charIter.Current, charIndex++ - 1); else buffer.Fallback(ch, charIndex - 1); if (fallback_chars == null || fallback_chars.Length < buffer.Remaining) fallback_chars = new char[buffer.Remaining]; for (var i = 0; i < fallback_chars.Length; i++) fallback_chars[i] = buffer.GetNextChar(); byteIndex += GetBytes(fallback_chars, 0, fallback_chars.Length, bytes, byteIndex, ref buffer, ref fallback_chars); } } } return charCount; }
public override int GetByteCount(char[] chars, int index, int count, bool flush) { if (chars == null) { throw new ArgumentNullException("chars"); } if (index < 0 || index > chars.Length) { throw new ArgumentOutOfRangeException("index"); } if (count < 0) { throw new ArgumentOutOfRangeException("count"); } if (index + count > chars.Length) { throw new ArgumentOutOfRangeException("chars"); } // The fallbackBuffer is created on-demand. The instance // FallbackBuffer isn't used because it wouldn't be thread safe. EncoderFallbackBuffer fallbackBuffer = null; char highSurrogate = HighSurrogate; int ret = 0; int count2 = index + count; for (; index < count2; index++) { char ch = chars[index]; if (highSurrogate != 0) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } // If we have a High/Low surrogates couple, we pass them // together if (char.IsLowSurrogate(ch)) { if (fallbackBuffer.Fallback(highSurrogate, ch, index - 1)) { HandleFallbackCount(fallbackBuffer, ref ret); } highSurrogate = '\0'; continue; } else { // First we pass the High surrogate to the Fallback if (fallbackBuffer.Fallback(highSurrogate, index - 1)) { HandleFallbackCount(fallbackBuffer, ref ret); } highSurrogate = '\0'; // Then we fall-through normal handling } } if (ch < VISCIIs.Length && (VISCIIs[ch] != 0 || ch == '\0')) { ret++; } else { // High/low surrogate handling, done through buffer if (char.IsHighSurrogate(ch)) { highSurrogate = ch; } else { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } // Fallback if (fallbackBuffer.Fallback(ch, index)) { HandleFallbackCount(fallbackBuffer, ref ret); } } } } if (flush) { if (highSurrogate != 0) { if (fallbackBuffer == null) { fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer(); } if (fallbackBuffer.Fallback(highSurrogate, index - 1)) { HandleFallbackCount(fallbackBuffer, ref ret); } } } return(ret); }