public static void DecoderExceptionFallbackBufferTest()
{
Decoder decoder = Encoding.GetEncoding("us-ascii", new EncoderExceptionFallback(), new DecoderExceptionFallback()).GetDecoder();
byte [] bytes = new byte [] { 0xFF, 0xFF };
char [] chars = new char [bytes.Length];
Assert.Throws <DecoderFallbackException>(() => decoder.GetChars(bytes, 0, 2, chars, 2));
DecoderFallbackBuffer fallbackBuffer = decoder.FallbackBuffer;
Assert.True(fallbackBuffer is DecoderExceptionFallbackBuffer, "Expected to get DecoderExceptionFallbackBuffer type");
Assert.Throws <DecoderFallbackException>(() => fallbackBuffer.Fallback(bytes, 0));
Assert.Throws <DecoderFallbackException>(() => fallbackBuffer.Fallback(new byte [] { 0x40, 0x60 }, 0));
Assert.Equal(0, fallbackBuffer.Remaining);
Assert.Equal('\u0000', fallbackBuffer.GetNextChar());
Assert.False(fallbackBuffer.MovePrevious(), "MovePrevious expected to always return false");
fallbackBuffer.Reset();
Assert.Equal(0, fallbackBuffer.Remaining);
Assert.Equal('\u0000', fallbackBuffer.GetNextChar());
}
public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex)
{
int byteEnd = byteIndex + byteCount;
int outputChars = 0;
while (byteIndex < byteEnd)
{
byte b = bytes[byteIndex];
char val;
if (!_map.Mapping.TryGetValue(b, out val))
{
DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
if (dfb.Fallback(new[] { b }, 0))
{
while (dfb.Remaining != 0)
{
chars[charIndex++] = (char)((int)_map.Mapping[(int)dfb.GetNextChar()]);
outputChars++;
}
}
}
else
{
chars[charIndex++] = val;
outputChars++;
}
byteIndex++;
}
return(outputChars);
}
public override int GetCharCount(byte[] bytes, int index, int count)
{
int byteEnd = index + count;
int outputChars = 0;
while (index < byteEnd)
{
byte b = bytes[index];
if (!_map.Mapping.TryGetValue(b, out _))
{
DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
if (dfb.Fallback(new[] { b }, 0))
{
outputChars += dfb.Remaining;
}
}
else
{
outputChars++;
}
index++;
}
return(outputChars);
}
public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex)
{
int byteEnd = byteIndex + byteCount;
int outputChars = 0;
while (byteIndex < byteEnd)
{
byte b = bytes[byteIndex];
if (b > 0x7f)
{
DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
if (dfb.Fallback(new byte[] { b }, byteIndex))
{
while (dfb.Remaining != 0)
{
chars[charIndex++] = dfb.GetNextChar();
outputChars++;
}
}
}
else
{
chars[charIndex++] = (char)b;
outputChars++;
}
byteIndex++;
}
return(outputChars);
}
public override int GetCharCount(byte[] bytes, int index, int count)
{
int byteEnd = index + count;
int outputChars = 0;
while (index < byteEnd)
{
byte b = bytes[index];
#if FEATURE_ENCODING
if (b > 0x7f)
{
DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
try {
if (dfb.Fallback(new[] { b }, 0))
{
outputChars += dfb.Remaining;
}
} catch (DecoderFallbackException ex) {
var dfe = new DecoderFallbackException("ordinal out of range(128)", ex.BytesUnknown, ex.Index);
dfe.Data.Add("encoding", EncodingName);
throw dfe;
}
}
else
{
outputChars++;
}
#else
outputChars++;
#endif
index++;
}
return(outputChars);
}
public override int GetCharCount(byte[] bytes, int index, int count)
{
int byteEnd = index + count;
int outputChars = 0;
while (index < byteEnd)
{
byte b = bytes[index];
#if FEATURE_ENCODING
if (b > 0x7f)
{
DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
if (dfb.Fallback(new[] { b }, 0))
{
outputChars += dfb.Remaining;
}
}
else
{
outputChars++;
}
#else
outputChars++;
#endif
index++;
}
return(outputChars);
}
private int GetCharCount(byte[] bytes, int index, int count, DecoderFallbackBuffer dfb)
{
int byteEnd = index + count;
int outputChars = 0;
while (index < byteEnd)
{
byte b = bytes[index];
if (b > 0x7f)
{
if (dfb == null)
{
dfb = DecoderFallback.CreateFallbackBuffer();
}
try {
if (dfb.Fallback(new[] { b }, index))
{
outputChars += dfb.Remaining;
while (dfb.GetNextChar() != char.MinValue) /* empty */ } {
}
} catch (DecoderFallbackException ex) {
var dfe = new DecoderFallbackException("ordinal not in range(128)", ex.BytesUnknown, ex.Index);
dfe.Data.Add("encoding", EncodingName);
throw dfe;
}
}
else
{
outputChars++;
}
index++;
}
return(outputChars);
}
void DecodeInvalidBytes(Stream stream, Action <Stream, string> decodedText, params byte[] data)
{
DecoderFallbackBuffer fallback = DecoderFallback.CreateFallbackBuffer();
bool result = fallback.Fallback(data, 0);
while (result && fallback.Remaining > 0)
{
decodedText(stream, $"{fallback.GetNextChar()}");
}
}
public override bool Fallback(byte[] bytesUnknown, int index)
{
if (bytesUnknown.Length + index == _parent._bufferLength)
{
_parentBuffer = null;
_parent._bytesUnknown = bytesUnknown;
return(false);
}
if (_parentBuffer == null)
{
_parentBuffer = _parent._parent.CreateFallbackBuffer();
}
return(_parentBuffer.Fallback(bytesUnknown, index));
}
public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex)
{
int byteEnd = byteIndex + byteCount;
int outputChars = 0;
while (byteIndex < byteEnd)
{
byte b = bytes[byteIndex];
object val;
object obj = ScriptingRuntimeHelpers.Int32ToObject((int)b);
if (!_map.TryGetValue(obj, out val) || val == null)
{
DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
if (dfb.Fallback(new[] { b }, 0))
{
while (dfb.Remaining != 0)
{
chars[charIndex++] = dfb.GetNextChar();
outputChars++;
}
}
}
else if (val is string)
{
string v = val as string;
for (int i = 0; i < v.Length; i++)
{
chars[charIndex++] = v[i];
outputChars++;
}
}
else if (val is int)
{
chars[charIndex++] = (char)(int)val;
outputChars++;
}
else
{
throw PythonOps.TypeError("charmap must be an int, str, or None");
}
byteIndex++;
}
return(outputChars);
}
private int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex, DecoderFallbackBuffer dfb)
{
int byteEnd = byteIndex + byteCount;
int outputChars = 0;
while (byteIndex < byteEnd)
{
byte b = bytes[byteIndex];
#if FEATURE_ENCODING
if (b > 0x7f)
{
if (dfb == null)
{
dfb = DecoderFallback.CreateFallbackBuffer();
}
try {
if (dfb.Fallback(new[] { b }, byteIndex))
{
while (dfb.Remaining != 0)
{
chars[charIndex++] = dfb.GetNextChar();
outputChars++;
}
}
} catch (DecoderFallbackException ex) {
var dfe = new DecoderFallbackException("ordinal not in range(128)", ex.BytesUnknown, ex.Index);
dfe.Data.Add("encoding", EncodingName);
throw dfe;
}
}
else
{
chars[charIndex++] = (char)b;
outputChars++;
}
#else
chars[charIndex++] = (char)b;
outputChars++;
#endif
byteIndex++;
}
return(outputChars);
}
public static void DecoderReplacementFallbackBufferTest()
{
Decoder decoder = Encoding.GetEncoding("us-ascii", new EncoderReplacementFallback(), new DecoderReplacementFallback()).GetDecoder();
byte [] bytes = new byte [] { 0xFF, 0xFF };
char [] chars = new char [bytes.Length];
DecoderFallbackBuffer fallbackBuffer = decoder.FallbackBuffer;
Assert.True(fallbackBuffer is DecoderReplacementFallbackBuffer, "Expected to get DecoderReplacementFallbackBuffer type");
Assert.True(fallbackBuffer.Fallback(bytes, 0), "Expected we fallback on the given buffer");
Assert.Equal(1, fallbackBuffer.Remaining);
Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are at the Beginning of the buffer");
Assert.Equal('?', fallbackBuffer.GetNextChar());
Assert.True(fallbackBuffer.MovePrevious(), "Expected we can move back on the replacement buffer");
Assert.Equal('?', fallbackBuffer.GetNextChar());
fallbackBuffer.Reset();
Assert.Equal(0, fallbackBuffer.Remaining);
Assert.Equal('\u0000', fallbackBuffer.GetNextChar());
Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are rest the buffer");
}
public override int GetCharCount(byte[] bytes, int index, int count)
{
int byteEnd = index + count;
int outputChars = 0;
while (index < byteEnd)
{
byte b = bytes[index];
object val;
object byteObj = ScriptingRuntimeHelpers.Int32ToObject((int)b);
if (!_map.TryGetValue(byteObj, out val) || val == null)
{
DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
if (dfb.Fallback(new[] { b }, 0))
{
outputChars += dfb.Remaining;
}
}
else if (val is string)
{
outputChars += ((string)val).Length;
}
else if (val is int)
{
outputChars++;
}
else
{
throw PythonOps.TypeError("charmap must be an int, str, or None");
}
index++;
}
return(outputChars);
}
int GetChars(byte[] bytes, int byteIndex, int byteCount,
char[] chars, int charIndex,
ref DecoderFallbackBuffer buffer)
{
if (bytes == null)
throw new ArgumentNullException("bytes");
if (chars == null)
throw new ArgumentNullException("chars");
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException("byteIndex");
if (byteCount < 0 || byteCount > (bytes.Length - byteIndex))
throw new ArgumentOutOfRangeException("byteCount");
if (charIndex < 0 || charIndex > chars.Length)
throw new ArgumentOutOfRangeException("charIndex");
if ((chars.Length - charIndex) < byteCount)
throw new ArgumentException("Insufficient Space");
var count = byteCount;
while (count-- > 0)
{
var c = (char) bytes[byteIndex++];
if (c < '\x80')
chars[charIndex++] = c;
else
{
if (buffer == null)
buffer = DecoderFallback.CreateFallbackBuffer();
var thisByte = new byte[] {bytes[byteIndex - 1]};
buffer.Fallback(thisByte, 0);
while (buffer.Remaining > 0)
{
if (charIndex < chars.Length)
{
chars[charIndex++] = buffer.GetNextChar();
continue;
}
throw new ArgumentException(
"The output char buffer is too small to contain the " +
"decoded characters.");
}
}
}
return byteCount;
}
public override bool Fallback(byte [] bytesUnknown, int index)
{
fallbackAction(bytesUnknown, index);
return(buffer.Fallback(bytesUnknown, index));
}
private int GetCharsInternal(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex)
{
var outpos = byteIndex;
var escape = false;
for (var inpos = byteIndex; inpos < (byteIndex + byteCount); inpos++)
{
var @byte = bytes[inpos];
var codepoint = NOCODE;
var extended = false;
if (escape)
{
if (@byte == 0x65)
{
codepoint = '\x20AC';
}
else
{
codepoint = GsmToUcs2Extended[@byte];
}
// If char is not a valid codepoint, use NBSP.
if (codepoint == NOCODE)
{
codepoint = '\xA0';
}
extended = true;
escape = false;
}
else if (@byte < GsmToUcs2.Length)
{
codepoint = GsmToUcs2[@byte];
if (codepoint == UESCAPE)
{
escape = true;
continue;
}
}
if (codepoint == NOCODE)
{
char tmp;
DecoderFallbackBuffer.Fallback(extended ? new byte[] { 0x1b, @byte } : new[] { @byte }, inpos);
while ((tmp = DecoderFallbackBuffer.GetNextChar()) != 0)
{
if (chars != null)
{
chars[outpos++] = GsmToUcs2[tmp]; // FIXME: Character might not be a 7-bit one..
}
else
{
outpos++;
}
}
}
else
{
if (chars != null)
{
chars[outpos++] = codepoint;
}
else
{
outpos += 1;
}
}
}
return(outpos - charIndex);
}