public static void DecoderExceptionFallbackBufferTest() { Decoder decoder = Encoding.GetEncoding("us-ascii", new EncoderExceptionFallback(), new DecoderExceptionFallback()).GetDecoder(); byte [] bytes = new byte [] { 0xFF, 0xFF }; char [] chars = new char [bytes.Length]; Assert.Throws <DecoderFallbackException>(() => decoder.GetChars(bytes, 0, 2, chars, 2)); DecoderFallbackBuffer fallbackBuffer = decoder.FallbackBuffer; Assert.True(fallbackBuffer is DecoderExceptionFallbackBuffer, "Expected to get DecoderExceptionFallbackBuffer type"); Assert.Throws <DecoderFallbackException>(() => fallbackBuffer.Fallback(bytes, 0)); Assert.Throws <DecoderFallbackException>(() => fallbackBuffer.Fallback(new byte [] { 0x40, 0x60 }, 0)); Assert.Equal(0, fallbackBuffer.Remaining); Assert.Equal('\u0000', fallbackBuffer.GetNextChar()); Assert.False(fallbackBuffer.MovePrevious(), "MovePrevious expected to always return false"); fallbackBuffer.Reset(); Assert.Equal(0, fallbackBuffer.Remaining); Assert.Equal('\u0000', fallbackBuffer.GetNextChar()); }
public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) { int byteEnd = byteIndex + byteCount; int outputChars = 0; while (byteIndex < byteEnd) { byte b = bytes[byteIndex]; char val; if (!_map.Mapping.TryGetValue(b, out val)) { DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer(); if (dfb.Fallback(new[] { b }, 0)) { while (dfb.Remaining != 0) { chars[charIndex++] = (char)((int)_map.Mapping[(int)dfb.GetNextChar()]); outputChars++; } } } else { chars[charIndex++] = val; outputChars++; } byteIndex++; } return(outputChars); }
public override int GetCharCount(byte[] bytes, int index, int count) { int byteEnd = index + count; int outputChars = 0; while (index < byteEnd) { byte b = bytes[index]; if (!_map.Mapping.TryGetValue(b, out _)) { DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer(); if (dfb.Fallback(new[] { b }, 0)) { outputChars += dfb.Remaining; } } else { outputChars++; } index++; } return(outputChars); }
public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) { int byteEnd = byteIndex + byteCount; int outputChars = 0; while (byteIndex < byteEnd) { byte b = bytes[byteIndex]; if (b > 0x7f) { DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer(); if (dfb.Fallback(new byte[] { b }, byteIndex)) { while (dfb.Remaining != 0) { chars[charIndex++] = dfb.GetNextChar(); outputChars++; } } } else { chars[charIndex++] = (char)b; outputChars++; } byteIndex++; } return(outputChars); }
public override int GetCharCount(byte[] bytes, int index, int count) { int byteEnd = index + count; int outputChars = 0; while (index < byteEnd) { byte b = bytes[index]; #if FEATURE_ENCODING if (b > 0x7f) { DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer(); try { if (dfb.Fallback(new[] { b }, 0)) { outputChars += dfb.Remaining; } } catch (DecoderFallbackException ex) { var dfe = new DecoderFallbackException("ordinal out of range(128)", ex.BytesUnknown, ex.Index); dfe.Data.Add("encoding", EncodingName); throw dfe; } } else { outputChars++; } #else outputChars++; #endif index++; } return(outputChars); }
public override int GetCharCount(byte[] bytes, int index, int count) { int byteEnd = index + count; int outputChars = 0; while (index < byteEnd) { byte b = bytes[index]; #if FEATURE_ENCODING if (b > 0x7f) { DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer(); if (dfb.Fallback(new[] { b }, 0)) { outputChars += dfb.Remaining; } } else { outputChars++; } #else outputChars++; #endif index++; } return(outputChars); }
private int GetCharCount(byte[] bytes, int index, int count, DecoderFallbackBuffer dfb) { int byteEnd = index + count; int outputChars = 0; while (index < byteEnd) { byte b = bytes[index]; if (b > 0x7f) { if (dfb == null) { dfb = DecoderFallback.CreateFallbackBuffer(); } try { if (dfb.Fallback(new[] { b }, index)) { outputChars += dfb.Remaining; while (dfb.GetNextChar() != char.MinValue) /* empty */ } { } } catch (DecoderFallbackException ex) { var dfe = new DecoderFallbackException("ordinal not in range(128)", ex.BytesUnknown, ex.Index); dfe.Data.Add("encoding", EncodingName); throw dfe; } } else { outputChars++; } index++; } return(outputChars); }
void DecodeInvalidBytes(Stream stream, Action <Stream, string> decodedText, params byte[] data) { DecoderFallbackBuffer fallback = DecoderFallback.CreateFallbackBuffer(); bool result = fallback.Fallback(data, 0); while (result && fallback.Remaining > 0) { decodedText(stream, $"{fallback.GetNextChar()}"); } }
public override bool Fallback(byte[] bytesUnknown, int index) { if (bytesUnknown.Length + index == _parent._bufferLength) { _parentBuffer = null; _parent._bytesUnknown = bytesUnknown; return(false); } if (_parentBuffer == null) { _parentBuffer = _parent._parent.CreateFallbackBuffer(); } return(_parentBuffer.Fallback(bytesUnknown, index)); }
public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) { int byteEnd = byteIndex + byteCount; int outputChars = 0; while (byteIndex < byteEnd) { byte b = bytes[byteIndex]; object val; object obj = ScriptingRuntimeHelpers.Int32ToObject((int)b); if (!_map.TryGetValue(obj, out val) || val == null) { DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer(); if (dfb.Fallback(new[] { b }, 0)) { while (dfb.Remaining != 0) { chars[charIndex++] = dfb.GetNextChar(); outputChars++; } } } else if (val is string) { string v = val as string; for (int i = 0; i < v.Length; i++) { chars[charIndex++] = v[i]; outputChars++; } } else if (val is int) { chars[charIndex++] = (char)(int)val; outputChars++; } else { throw PythonOps.TypeError("charmap must be an int, str, or None"); } byteIndex++; } return(outputChars); }
private int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex, DecoderFallbackBuffer dfb) { int byteEnd = byteIndex + byteCount; int outputChars = 0; while (byteIndex < byteEnd) { byte b = bytes[byteIndex]; #if FEATURE_ENCODING if (b > 0x7f) { if (dfb == null) { dfb = DecoderFallback.CreateFallbackBuffer(); } try { if (dfb.Fallback(new[] { b }, byteIndex)) { while (dfb.Remaining != 0) { chars[charIndex++] = dfb.GetNextChar(); outputChars++; } } } catch (DecoderFallbackException ex) { var dfe = new DecoderFallbackException("ordinal not in range(128)", ex.BytesUnknown, ex.Index); dfe.Data.Add("encoding", EncodingName); throw dfe; } } else { chars[charIndex++] = (char)b; outputChars++; } #else chars[charIndex++] = (char)b; outputChars++; #endif byteIndex++; } return(outputChars); }
public static void DecoderReplacementFallbackBufferTest() { Decoder decoder = Encoding.GetEncoding("us-ascii", new EncoderReplacementFallback(), new DecoderReplacementFallback()).GetDecoder(); byte [] bytes = new byte [] { 0xFF, 0xFF }; char [] chars = new char [bytes.Length]; DecoderFallbackBuffer fallbackBuffer = decoder.FallbackBuffer; Assert.True(fallbackBuffer is DecoderReplacementFallbackBuffer, "Expected to get DecoderReplacementFallbackBuffer type"); Assert.True(fallbackBuffer.Fallback(bytes, 0), "Expected we fallback on the given buffer"); Assert.Equal(1, fallbackBuffer.Remaining); Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are at the Beginning of the buffer"); Assert.Equal('?', fallbackBuffer.GetNextChar()); Assert.True(fallbackBuffer.MovePrevious(), "Expected we can move back on the replacement buffer"); Assert.Equal('?', fallbackBuffer.GetNextChar()); fallbackBuffer.Reset(); Assert.Equal(0, fallbackBuffer.Remaining); Assert.Equal('\u0000', fallbackBuffer.GetNextChar()); Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are rest the buffer"); }
public override int GetCharCount(byte[] bytes, int index, int count) { int byteEnd = index + count; int outputChars = 0; while (index < byteEnd) { byte b = bytes[index]; object val; object byteObj = ScriptingRuntimeHelpers.Int32ToObject((int)b); if (!_map.TryGetValue(byteObj, out val) || val == null) { DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer(); if (dfb.Fallback(new[] { b }, 0)) { outputChars += dfb.Remaining; } } else if (val is string) { outputChars += ((string)val).Length; } else if (val is int) { outputChars++; } else { throw PythonOps.TypeError("charmap must be an int, str, or None"); } index++; } return(outputChars); }
int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex, ref DecoderFallbackBuffer buffer) { if (bytes == null) throw new ArgumentNullException("bytes"); if (chars == null) throw new ArgumentNullException("chars"); if (byteIndex < 0 || byteIndex > bytes.Length) throw new ArgumentOutOfRangeException("byteIndex"); if (byteCount < 0 || byteCount > (bytes.Length - byteIndex)) throw new ArgumentOutOfRangeException("byteCount"); if (charIndex < 0 || charIndex > chars.Length) throw new ArgumentOutOfRangeException("charIndex"); if ((chars.Length - charIndex) < byteCount) throw new ArgumentException("Insufficient Space"); var count = byteCount; while (count-- > 0) { var c = (char) bytes[byteIndex++]; if (c < '\x80') chars[charIndex++] = c; else { if (buffer == null) buffer = DecoderFallback.CreateFallbackBuffer(); var thisByte = new byte[] {bytes[byteIndex - 1]}; buffer.Fallback(thisByte, 0); while (buffer.Remaining > 0) { if (charIndex < chars.Length) { chars[charIndex++] = buffer.GetNextChar(); continue; } throw new ArgumentException( "The output char buffer is too small to contain the " + "decoded characters."); } } } return byteCount; }
public override bool Fallback(byte [] bytesUnknown, int index) { fallbackAction(bytesUnknown, index); return(buffer.Fallback(bytesUnknown, index)); }
private int GetCharsInternal(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) { var outpos = byteIndex; var escape = false; for (var inpos = byteIndex; inpos < (byteIndex + byteCount); inpos++) { var @byte = bytes[inpos]; var codepoint = NOCODE; var extended = false; if (escape) { if (@byte == 0x65) { codepoint = '\x20AC'; } else { codepoint = GsmToUcs2Extended[@byte]; } // If char is not a valid codepoint, use NBSP. if (codepoint == NOCODE) { codepoint = '\xA0'; } extended = true; escape = false; } else if (@byte < GsmToUcs2.Length) { codepoint = GsmToUcs2[@byte]; if (codepoint == UESCAPE) { escape = true; continue; } } if (codepoint == NOCODE) { char tmp; DecoderFallbackBuffer.Fallback(extended ? new byte[] { 0x1b, @byte } : new[] { @byte }, inpos); while ((tmp = DecoderFallbackBuffer.GetNextChar()) != 0) { if (chars != null) { chars[outpos++] = GsmToUcs2[tmp]; // FIXME: Character might not be a 7-bit one.. } else { outpos++; } } } else { if (chars != null) { chars[outpos++] = codepoint; } else { outpos += 1; } } } return(outpos - charIndex); }