public static void EncoderExceptionFallbackBufferTest()
{
Encoder encoder = Encoding.GetEncoding("us-ascii", new EncoderExceptionFallback(), new DecoderExceptionFallback()).GetEncoder();
char [] chars = new char[] { '\uFFFF' };
byte [] bytes = new byte[2];
Assert.Throws <EncoderFallbackException>(() => encoder.GetBytes(chars, 0, 1, bytes, 0, true));
EncoderFallbackBuffer fallbackBuffer = encoder.FallbackBuffer;
Assert.True(fallbackBuffer is EncoderExceptionFallbackBuffer, "Expected to get EncoderExceptionFallbackBuffer type");
Assert.Throws <EncoderFallbackException>(() => fallbackBuffer.Fallback(chars[0], 0));
Assert.Throws <EncoderFallbackException>(() => fallbackBuffer.Fallback('\u0040', 0));
Assert.Throws <EncoderFallbackException>(() => fallbackBuffer.Fallback('\uD800', '\uDC00', 0));
Assert.Equal(0, fallbackBuffer.Remaining);
Assert.Equal('\u0000', fallbackBuffer.GetNextChar());
Assert.False(fallbackBuffer.MovePrevious(), "MovePrevious expected to always return false");
fallbackBuffer.Reset();
Assert.Equal(0, fallbackBuffer.Remaining);
Assert.Equal('\u0000', fallbackBuffer.GetNextChar());
}
private int GetByteCount(char[] chars, int index, int count, EncoderFallbackBuffer efb)
{
int byteCount = 0;
int charEnd = index + count;
while (index < charEnd)
{
char c = chars[index];
if (c > 0x7f)
{
if (efb == null)
{
efb = EncoderFallback.CreateFallbackBuffer();
}
if (efb.Fallback(c, index))
{
byteCount += efb.Remaining;
while (efb.GetNextChar() != char.MinValue) /* empty */ } {
}
}
else
{
byteCount++;
}
index++;
}
return(byteCount);
}
public void HandleFallback(ref EncoderFallbackBuffer buffer,
char[] chars, ref int charIndex, ref int charCount,
byte[] bytes, ref int byteIndex, ref int byteCount, object state)
{
if (buffer == null)
{
buffer = EncoderFallback.CreateFallbackBuffer();
}
if (charCount > 1 && (Char.IsSurrogate(chars[charIndex]) && Char.IsSurrogate(chars[charIndex + 1])))
{
buffer.Fallback(chars[charIndex], chars[charIndex + 1], charIndex);
charIndex++;
charCount--;
}
else
{
buffer.Fallback(chars[charIndex], charIndex);
}
char[] tmp = new char[buffer.Remaining];
int idx = 0;
while (buffer.Remaining > 0)
{
tmp[idx++] = buffer.GetNextChar();
}
var len = state == null?
GetBytes(tmp, 0, tmp.Length, bytes, byteIndex)
: GetBytesInternal(tmp, 0, tmp.Length, bytes, byteIndex, true, state);
byteIndex += len;
byteCount -= len;
}
public void HandleFallback(ref EncoderFallbackBuffer buffer,
char[] chars, ref int charIndex, ref int charCount,
byte[] bytes, ref int byteIndex, ref int byteCount, object state)
{
if (buffer == null)
buffer = EncoderFallback.CreateFallbackBuffer();
if (charCount > 1 && (Char.IsSurrogate(chars[charIndex]) && Char.IsSurrogate(chars[charIndex + 1])))
{
buffer.Fallback (chars[charIndex], chars[charIndex + 1], charIndex);
charIndex++;
charCount--;
}
else
buffer.Fallback (chars[charIndex], charIndex);
char[] tmp = new char[buffer.Remaining];
int idx = 0;
while (buffer.Remaining > 0)
tmp[idx++] = buffer.GetNextChar();
var len = state == null ?
GetBytes(tmp, 0, tmp.Length, bytes, byteIndex)
: GetBytesInternal(tmp, 0, tmp.Length, bytes, byteIndex, true, state);
byteIndex += len;
byteCount -= len;
}
public unsafe void HandleFallback(ref EncoderFallbackBuffer buffer,
char *chars, ref int charIndex, ref int charCount,
byte *bytes, ref int byteIndex, ref int byteCount)
{
if (buffer == null)
{
buffer = EncoderFallback.CreateFallbackBuffer();
}
if (Char.IsSurrogate(chars [charIndex]) && charCount > 0 &&
Char.IsSurrogate(chars [charIndex + 1]))
{
buffer.Fallback(chars [charIndex], chars [charIndex + 1], charIndex);
charIndex++;
charCount--;
}
else
{
buffer.Fallback(chars [charIndex], charIndex);
}
char [] tmp = new char [buffer.Remaining];
int idx = 0;
while (buffer.Remaining > 0)
tmp [idx++] = buffer.GetNextChar();
fixed(char *tmparr = tmp)
{
byteIndex += GetBytes(tmparr, tmp.Length, bytes + byteIndex, byteCount);
}
}
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex)
{
int charEnd = charIndex + charCount;
int outputBytes = 0;
while (charIndex < charEnd)
{
char c = chars[charIndex];
char val;
if (!_map.Mapping.TryGetValue((int)c, out val))
{
EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer();
if (efb.Fallback(c, charIndex))
{
while (efb.Remaining != 0)
{
bytes[byteIndex++] = (byte)_map.Mapping[(int)efb.GetNextChar()];
outputBytes++;
}
}
}
else
{
bytes[byteIndex++] = (byte)val;
outputBytes++;
}
charIndex++;
}
return(outputBytes);
}
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex)
{
int charEnd = charIndex + charCount;
int outputBytes = 0;
while (charIndex < charEnd)
{
char c = chars[charIndex];
if (c > 0x7f)
{
EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer();
if (efb.Fallback(c, charIndex))
{
while (efb.Remaining != 0)
{
bytes[byteIndex++] = (byte)efb.GetNextChar();
outputBytes++;
}
}
}
else
{
bytes[byteIndex++] = (byte)c;
outputBytes++;
}
charIndex++;
}
return(outputBytes);
}
public override char GetNextChar()
{
if (!Data.HasValue)
{
return(LastResortEncoderFallbackBuffer.GetNextChar());
}
return(Pos++ == 0 ? Data.Value : '\0');
}
public static void EncoderReplacementFallbackBufferTest()
{
Encoder encoder = Encoding.GetEncoding("us-ascii", new EncoderReplacementFallback(), new DecoderReplacementFallback()).GetEncoder();
char [] chars = new char [] { '\uFFFF' };
byte [] bytes = new byte [2];
EncoderFallbackBuffer fallbackBuffer = encoder.FallbackBuffer;
Assert.True(fallbackBuffer is EncoderReplacementFallbackBuffer, "Expected to get EncoderReplacementFallbackBuffer type");
Assert.True(fallbackBuffer.Fallback(chars[0], 0), "Expected we fallback on the given buffer");
Assert.Equal(1, fallbackBuffer.Remaining);
Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are at the Beginning of the buffer");
Assert.Equal('?', fallbackBuffer.GetNextChar());
Assert.True(fallbackBuffer.MovePrevious(), "Expected we can move back on the replacement buffer");
Assert.Equal('?', fallbackBuffer.GetNextChar());
fallbackBuffer.Reset();
Assert.Equal(0, fallbackBuffer.Remaining);
Assert.Equal('\u0000', fallbackBuffer.GetNextChar());
Assert.False(fallbackBuffer.MovePrevious(), "Expected we cannot move back on the replacement buffer as we are rest the buffer");
}
private static void HandleFallback(EncoderFallbackBuffer fallbackBuffer, byte[] bytes, ref int byteIndex)
{
while (fallbackBuffer.Remaining > 0)
{
char ch = fallbackBuffer.GetNextChar();
byte b;
// chars between 31 and 127 are equal in Unicode and VISCII
if (!CharToByte(ch, out b))
{
throw new EncoderFallbackException();
}
WriteByte(bytes, ref byteIndex, b);
}
}
protected static void HandleFallbackCount(EncoderFallbackBuffer fallbackBuffer, ref int count)
{
while (fallbackBuffer.Remaining > 0)
{
char ch = fallbackBuffer.GetNextChar();
if (!(ch < VISCIIs.Length && (VISCIIs[ch] != 0 || ch == '\0')))
{
throw new EncoderFallbackException();
}
count++;
}
}
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex)
{
int charEnd = charIndex + charCount;
int outputBytes = 0;
while (charIndex < charEnd)
{
char c = chars[charIndex];
object val;
object obj = (int)c;
if (!_map.TryGetValue(obj, out val) || (val == null && _errors == "strict"))
{
EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer();
if (efb.Fallback(c, charIndex))
{
while (efb.Remaining != 0)
{
obj = (int)efb.GetNextChar();
bytes[byteIndex++] = (byte)((int)_map[obj]);
outputBytes++;
}
}
}
else if (val == null)
{
throw PythonOps.UnicodeEncodeError("charmap", c.ToString(), charIndex, charIndex + 1, "character maps to <undefined>");
}
else if (val is string)
{
string v = val as string;
for (int i = 0; i < v.Length; i++)
{
bytes[byteIndex++] = (byte)v[i];
outputBytes++;
}
}
else if (val is int)
{
bytes[byteIndex++] = (byte)(int)val;
outputBytes++;
}
else
{
throw PythonOps.TypeError("charmap must be an int, str, or None");
}
charIndex++;
}
return(outputBytes);
}
protected static void HandleFallbackWrite(EncoderFallbackBuffer fallbackBuffer, byte[] bytes, ref int byteIndex)
{
while (fallbackBuffer.Remaining > 0)
{
char ch = fallbackBuffer.GetNextChar();
byte b;
if (!(ch < VISCIIs.Length && ((b = VISCIIs[ch]) != 0 || ch == '\0')))
{
throw new EncoderFallbackException();
}
WriteByte(bytes, byteIndex, b);
byteIndex++;
}
}
private int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex, EncoderFallbackBuffer efb)
{
int charEnd = charIndex + charCount;
int outputBytes = 0;
while (charIndex < charEnd)
{
char c = chars[charIndex];
#if FEATURE_ENCODING
if (c > 0x7f)
{
if (efb == null)
{
efb = EncoderFallback.CreateFallbackBuffer();
}
if (efb.Fallback(c, charIndex))
{
while (efb.Remaining != 0)
{
bytes[byteIndex++] = (byte)efb.GetNextChar();
outputBytes++;
}
}
}
else
{
bytes[byteIndex++] = (byte)c;
outputBytes++;
}
#else
bytes[byteIndex++] = (byte)c;
outputBytes++;
#endif
charIndex++;
}
return(outputBytes);
}
public unsafe void HandleFallback(ref EncoderFallbackBuffer buffer,
char *chars, ref int charIndex, ref int charCount,
byte *bytes, ref int byteIndex, ref int byteCount, object state)
{
if (buffer == null)
{
buffer = EncoderFallback.CreateFallbackBuffer();
}
if (charCount > 1 && (Char.IsSurrogate(chars [charIndex]) && Char.IsSurrogate(chars [charIndex + 1])))
{
buffer.Fallback(chars [charIndex], chars [charIndex + 1], charIndex);
charIndex++;
charCount--;
}
else
{
buffer.Fallback(chars [charIndex], charIndex);
}
char [] tmp = new char [buffer.Remaining];
int idx = 0;
while (buffer.Remaining > 0)
tmp [idx++] = buffer.GetNextChar();
fixed(char *tmparr = tmp)
{
var outbytes = bytes == null ? null : bytes + byteIndex;
var len = state == null?
GetBytes(tmparr, tmp.Length, outbytes, byteCount)
: GetBytesInternal(tmparr, tmp.Length, outbytes, byteCount, true, state);
byteIndex += len;
byteCount -= len;
}
}
public override int GetByteCount(char[] chars, int index, int count, bool flush)
{
if (chars == null)
{
throw new ArgumentNullException("chars");
}
if (index < 0)
{
throw new ArgumentOutOfRangeException("index");
}
if (count < 0)
{
throw new ArgumentOutOfRangeException("count");
}
if (index + count > chars.Length)
{
throw new ArgumentOutOfRangeException("chars");
}
// The fallbackBuffer is created on-demand. The instance
// FallbackBuffer isn't used because it wouldn't be thread safe.
EncoderFallbackBuffer fallbackBuffer = null;
// Written only on flush = false
char?buffer = Buffer;
int ret = 0;
int count2 = index + count;
for (; index < count2; index++)
{
char ch = chars[index];
if (buffer != null)
{
// If we have a High/Low surrogates couple, we pass them
// together
if (char.IsLowSurrogate(ch))
{
if (fallbackBuffer == null)
{
fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer();
}
if (fallbackBuffer.Fallback(buffer.Value, ch, index - 1))
{
while (fallbackBuffer.Remaining > 0)
{
fallbackBuffer.GetNextChar();
ret++;
}
}
buffer = null;
continue;
}
else
{
if (fallbackBuffer == null)
{
fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer();
}
// First we pass the High surrogate to the Fallback
if (fallbackBuffer.Fallback(buffer.Value, index - 1))
{
while (fallbackBuffer.Remaining > 0)
{
fallbackBuffer.GetNextChar();
ret++;
}
}
buffer = null;
// Then we fall-through normal handling
}
}
// High/low surrogate handling, done through buffer
if (char.IsHighSurrogate(ch))
{
buffer = ch;
}
else
{
byte b;
if (!CharToByte(ch, out b))
{
if (fallbackBuffer == null)
{
fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer();
}
// Fallback
if (fallbackBuffer.Fallback(ch, index))
{
while (fallbackBuffer.Remaining > 0)
{
fallbackBuffer.GetNextChar();
ret++;
}
}
continue;
}
ret++;
}
}
if (flush)
{
if (buffer != null)
{
if (fallbackBuffer == null)
{
fallbackBuffer = (Fallback ?? EncoderFallback.ReplacementFallback).CreateFallbackBuffer();
}
if (fallbackBuffer.Fallback(buffer.Value, index - 1))
{
while (fallbackBuffer.Remaining > 0)
{
fallbackBuffer.GetNextChar();
ret++;
}
}
}
}
else
{
Buffer = buffer;
}
return(ret);
}
int InternalGetBytes(IEnumerable<char> chars, int charLength, int charIndex, int charCount,
byte[] bytes, int byteIndex,
ref EncoderFallbackBuffer buffer,
ref char[] fallback_chars)
{
if (bytes == null)
throw new ArgumentNullException("bytes");
if (charIndex < 0 || charIndex > charLength)
throw new ArgumentOutOfRangeException("charIndex");
if (charCount < 0 || charCount > (charLength - charIndex))
throw new ArgumentOutOfRangeException("charCount");
if (byteIndex < 0 || byteIndex > bytes.Length)
throw new ArgumentOutOfRangeException("byteIndex");
if ((bytes.Length - byteIndex) < charCount)
throw new ArgumentException("InsufficientSpace");
using (var charIter = chars.GetEnumerator())
{
for (var i = 0; i <= charIndex; ++i)
charIter.MoveNext();
var count = charCount;
while (count-- > 0)
{
var ch = charIter.Current;
++charIndex;
charIter.MoveNext();
if (ch < (char) 0x80)
{
bytes[byteIndex++] = (byte) ch;
}
else
{
if (buffer == null)
buffer = EncoderFallback.CreateFallbackBuffer();
if (Char.IsSurrogate(ch) && count > 1 &&
Char.IsSurrogate(charIter.Current))
buffer.Fallback(ch, charIter.Current, charIndex++ - 1);
else
buffer.Fallback(ch, charIndex - 1);
if (fallback_chars == null || fallback_chars.Length < buffer.Remaining)
fallback_chars = new char[buffer.Remaining];
for (var i = 0; i < fallback_chars.Length; i++)
fallback_chars[i] = buffer.GetNextChar();
byteIndex += GetBytes(fallback_chars, 0,
fallback_chars.Length, bytes, byteIndex,
ref buffer, ref fallback_chars);
}
}
}
return charCount;
}
private int GetBytesInternal(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex)
{
var outpos = byteIndex;
for (var inpos = charIndex; inpos < (charIndex + charCount); inpos++)
{
var character = chars[inpos];
var @byte = NOCHAR;
var escape = false;
if (character < Ucs2ToGsm.Length)
{
@byte = Ucs2ToGsm[character];
if (@byte == ESCAPE)
{
escape = true;
@byte = Ucs2ToGsmExtended[character];
}
}
else if (character >= Ucs2GclToGsmBase && character <= Ucs2GclToGsmMax)
{
escape = true;
@byte = Ucs2GclToGsm[character - Ucs2GclToGsmBase];
}
else if (character == '\x20AC') // Euro sign.
{
escape = true;
@byte = 0x65;
}
if (@byte == NOCHAR)
{
char tmp;
EncoderFallbackBuffer.Fallback(character, inpos);
while ((tmp = EncoderFallbackBuffer.GetNextChar()) != 0)
{
if (bytes != null)
{
bytes[outpos++] = Ucs2ToGsm[tmp]; // FIXME: Character might not be a 7-bit one..
}
else
{
outpos++;
}
}
EncoderFallbackBuffer.Reset();
}
else
{
if (bytes != null)
{
if (escape)
{
bytes[outpos++] = ESCAPE;
}
bytes[outpos++] = @byte;
}
else
{
outpos += escape ? 2 : 1;
}
}
}
return(outpos - byteIndex);
}
