public override int GetByteCount(char[] chars, int index, int count) { int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; if (!_map.Mapping.TryGetValue(c, out _)) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, index)) { byteCount += efb.Remaining; } } else { byteCount++; } index++; } return(byteCount); }
public unsafe void HandleFallback(ref EncoderFallbackBuffer buffer, char *chars, ref int charIndex, ref int charCount, byte *bytes, ref int byteIndex, ref int byteCount) { if (buffer == null) { buffer = EncoderFallback.CreateFallbackBuffer(); } if (Char.IsSurrogate(chars [charIndex]) && charCount > 0 && Char.IsSurrogate(chars [charIndex + 1])) { buffer.Fallback(chars [charIndex], chars [charIndex + 1], charIndex); charIndex++; charCount--; } else { buffer.Fallback(chars [charIndex], charIndex); } char [] tmp = new char [buffer.Remaining]; int idx = 0; while (buffer.Remaining > 0) tmp [idx++] = buffer.GetNextChar(); fixed(char *tmparr = tmp) { byteIndex += GetBytes(tmparr, tmp.Length, bytes + byteIndex, byteCount); } }
protected override int GetBytes(ReadOnlySpan <char> chars, Span <byte> bytes, bool write) { if (chars.IsEmpty) { return(0); } for (int i = 0; i < chars.Length; i++) { char c = chars[i]; if ((uint)c >= (uint)_isAllowed.Length || !_isAllowed[c]) { EncoderFallback.CreateFallbackBuffer().Fallback(c, i); Debug.Fail("Fallback should have thrown"); throw new CryptographicException(); } if (write) { bytes[i] = (byte)c; } } return(chars.Length); }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; char val; if (!_map.Mapping.TryGetValue((int)c, out val)) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { bytes[byteIndex++] = (byte)_map.Mapping[(int)efb.GetNextChar()]; outputBytes++; } } } else { bytes[byteIndex++] = (byte)val; outputBytes++; } charIndex++; } return(outputBytes); }
protected override int GetBytes(ReadOnlySpan <char> chars, Span <byte> bytes, bool write) { if (chars.IsEmpty) { return(0); } int writeIdx = 0; for (int i = 0; i < chars.Length; i++) { char c = chars[i]; if (char.IsSurrogate(c)) { EncoderFallback.CreateFallbackBuffer().Fallback(c, i); Debug.Fail("Fallback should have thrown"); throw new InvalidOperationException(); } ushort val16 = c; if (write) { bytes[writeIdx + 1] = (byte)val16; bytes[writeIdx] = (byte)(val16 >> 8); } writeIdx += 2; } return(writeIdx); }
private int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex, EncoderFallbackBuffer efb) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; if (c > 0x7f) { if (efb == null) { efb = EncoderFallback.CreateFallbackBuffer(); } if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { bytes[byteIndex++] = (byte)efb.GetNextChar(); outputBytes++; } } } else { bytes[byteIndex++] = (byte)c; outputBytes++; } charIndex++; } return(outputBytes); }
private int GetByteCount(char[] chars, int index, int count, EncoderFallbackBuffer efb) { int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; if (c > 0x7f) { if (efb == null) { efb = EncoderFallback.CreateFallbackBuffer(); } if (efb.Fallback(c, index)) { byteCount += efb.Remaining; while (efb.GetNextChar() != char.MinValue) /* empty */ } { } } else { byteCount++; } index++; } return(byteCount); }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; #if FEATURE_ENCODING if (c > 0x7f) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { bytes[byteIndex++] = (byte)efb.GetNextChar(); outputBytes++; } } } else { bytes[byteIndex++] = (byte)c; outputBytes++; } #else bytes[byteIndex++] = (byte)c; outputBytes++; #endif charIndex++; } return(outputBytes); }
public override int GetByteCount(char[] chars, int index, int count) { #if FEATURE_ENCODING int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; if (c > 0x7f) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, index)) { byteCount += efb.Remaining; } } else { byteCount++; } index++; } return(byteCount); #else return(count); #endif }
public override EncoderFallbackBuffer CreateFallbackBuffer() { // called by the decoder when it encounters // the first byte that it is unable to successfully decode WasTriggered = true; return(encoder_.CreateFallbackBuffer()); }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { int charEnd = charIndex + charCount; int outputBytes = 0; while (charIndex < charEnd) { char c = chars[charIndex]; object val; object obj = (int)c; if (!_map.TryGetValue(obj, out val) || (val == null && _errors == "strict")) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, charIndex)) { while (efb.Remaining != 0) { obj = (int)efb.GetNextChar(); bytes[byteIndex++] = (byte)((int)_map[obj]); outputBytes++; } } } else if (val == null) { throw PythonOps.UnicodeEncodeError("charmap", c.ToString(), charIndex, charIndex + 1, "character maps to <undefined>"); } else if (val is string) { string v = val as string; for (int i = 0; i < v.Length; i++) { bytes[byteIndex++] = (byte)v[i]; outputBytes++; } } else if (val is int) { bytes[byteIndex++] = (byte)(int)val; outputBytes++; } else { throw PythonOps.TypeError("charmap must be an int, str, or None"); } charIndex++; } return(outputBytes); }
public override int GetByteCount(char[] chars, int index, int count) { int byteCount = 0; int charEnd = index + count; while (index < charEnd) { char c = chars[index]; object val; object charObj = (int)c; if (!_map.TryGetValue(charObj, out val) || (val == null && _errors == "strict")) { EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer(); if (efb.Fallback(c, index)) { byteCount += efb.Remaining; } } else if (val == null) { throw PythonOps.UnicodeEncodeError("charmap", c.ToString(), index, index + 1, "character maps to <undefined>"); } else if (val is string) { byteCount += ((string)val).Length; } else if (val is int) { byteCount++; } else { throw PythonOps.TypeError("charmap must be an int, str, or None"); } index++; } return(byteCount); }
public unsafe void HandleFallback(ref EncoderFallbackBuffer buffer, char *chars, ref int charIndex, ref int charCount, byte *bytes, ref int byteIndex, ref int byteCount, object state) { if (buffer == null) { buffer = EncoderFallback.CreateFallbackBuffer(); } if (charCount > 1 && (Char.IsSurrogate(chars [charIndex]) && Char.IsSurrogate(chars [charIndex + 1]))) { buffer.Fallback(chars [charIndex], chars [charIndex + 1], charIndex); charIndex++; charCount--; } else { buffer.Fallback(chars [charIndex], charIndex); } char [] tmp = new char [buffer.Remaining]; int idx = 0; while (buffer.Remaining > 0) tmp [idx++] = buffer.GetNextChar(); fixed(char *tmparr = tmp) { var outbytes = bytes == null ? null : bytes + byteIndex; var len = state == null? GetBytes(tmparr, tmp.Length, outbytes, byteCount) : GetBytesInternal(tmparr, tmp.Length, outbytes, byteCount, true, state); byteIndex += len; byteCount -= len; } }
public void HandleFallback(ref EncoderFallbackBuffer buffer, char[] chars, ref int charIndex, ref int charCount, byte[] bytes, ref int byteIndex, ref int byteCount, object state) { if (buffer == null) { buffer = EncoderFallback.CreateFallbackBuffer(); } // THIS IS WERE THE BUG IS!! (pruiz) if (charCount > 1 && (Char.IsSurrogate(chars[charIndex]) && Char.IsSurrogate(chars[charIndex + 1]))) { buffer.Fallback(chars[charIndex], chars[charIndex + 1], charIndex); charIndex++; charCount--; } else { buffer.Fallback(chars[charIndex], charIndex); } char[] tmp = new char[buffer.Remaining]; int idx = 0; while (buffer.Remaining > 0) { tmp[idx++] = buffer.GetNextChar(); } var len = state == null? GetBytes(tmp, 0, tmp.Length, bytes, byteIndex) : GetBytesInternal(tmp, 0, tmp.Length, bytes, byteIndex, true, state); byteIndex += len; byteCount -= len; }
unsafe int InternalGetBytes(char *chars, int charLength, int charIndex, int charCount, byte[] bytes, int byteIndex, ref EncoderFallbackBuffer buffer, ref char[] fallback_chars) { if (bytes == null) { throw new ArgumentNullException("bytes"); } if (charIndex < 0 || charIndex > charLength) { throw new ArgumentOutOfRangeException("charIndex", "ArgRange_StringIndex"); } if (charCount < 0 || charCount > (charLength - charIndex)) { throw new ArgumentOutOfRangeException("charCount", "ArgRange_StringRange"); } if (byteIndex < 0 || byteIndex > bytes.Length) { throw new ArgumentOutOfRangeException("byteIndex", "ArgRange_Array"); } if ((bytes.Length - byteIndex) < charCount) { throw new ArgumentException("Arg_InsufficientSpace"); } int count = charCount; char ch; while (count-- > 0) { ch = chars[charIndex++]; if (ch < (char)0x80) { bytes[byteIndex++] = (byte)ch; } else { if (buffer == null) { buffer = EncoderFallback.CreateFallbackBuffer(); } if (Char.IsSurrogate(ch) && count > 1 && Char.IsSurrogate(chars[charIndex])) { buffer.Fallback(ch, chars[charIndex], charIndex++ - 1); } else { buffer.Fallback(ch, charIndex - 1); } if (fallback_chars == null || fallback_chars.Length < buffer.Remaining) { fallback_chars = new char[buffer.Remaining]; } for (int i = 0; i < fallback_chars.Length; i++) { fallback_chars[i] = buffer.GetNextChar(); } byteIndex += GetBytes(fallback_chars, 0, fallback_chars.Length, bytes, byteIndex, ref buffer, ref fallback_chars); } } return(charCount); }
public GSMBestFitEncoderFallbackBuffer(EncoderFallback caller, EncoderFallback lastResortFallback) { LastResortEncoderFallbackBuffer = lastResortFallback.CreateFallbackBuffer(); }
public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { System.Diagnostics.Debug.Assert(bytes.Length >= charCount); for (var index = charIndex; index < charIndex + charCount; index++) { var c = chars[index]; var code = (int)c; if (charToBytes_.ContainsKey(c)) { var b = charToBytes_[c]; bytes[charIndex + index] = b; continue; } else if (code <= 255) { var b = (byte)code; // codePoint is a byte excluding those that are already mapped if (!charToBytes_.Values.Contains(b)) { bytes[charIndex + index] = b; continue; } } // if no valid characters could be converted // use the configured fallback mechanism if (EncoderFallback != null) { System.Diagnostics.Debug.Assert(EncoderFallback.MaxCharCount == 1); var fallbackBuffer = EncoderFallback.CreateFallbackBuffer(); if (fallbackBuffer.Fallback(c, index)) { var next = fallbackBuffer.GetNextChar(); // recursively use this implementation // to convert the replacement char to // its corresponding byte var buffer = new byte[1]; if (GetBytes(new[] { next, }, 0, 1, buffer, 0) == 1) { var b = buffer[0]; bytes[charIndex + index] = b; continue; } // if the replacement character cannot // be mapped, simply fallback to a '?'. // (in practice, this should not happen) else { var b = (byte)'?'; bytes[charIndex + index] = b; continue; } } } // otherwise, just use a '?' character // (again, in practice, this should not // happen since a fallback should always // be configured on the Encoding. else { var b = (byte)'?'; bytes[charIndex + index] = b; } } return(charCount); }