/// <summary>A converter that processes a byte buffer containing a mix of UTF8 and Latin-1/Cp1252 chars.</summary> /// <remarks> /// A converter that processes a byte buffer containing a mix of UTF8 and Latin-1/Cp1252 chars. /// The result is a buffer where those chars have been converted to UTF-8; /// that means it contains only valid UTF-8 chars. /// <para /> /// <em>Explanation of the processing:</em> First the encoding of the buffer is detected looking /// at the first four bytes (that works only if the buffer starts with an ASCII-char, /// like xmls '<'). UTF-16/32 flavours do not require further proccessing. /// <para /> /// In the case, UTF-8 is detected, it assumes wrong UTF8 chars to be a sequence of /// Latin-1/Cp1252 encoded bytes and converts the chars to their corresponding UTF-8 byte /// sequence. /// <para /> /// The 0x80..0x9F range is undefined in Latin-1, but is defined in Windows code /// page 1252. The bytes 0x81, 0x8D, 0x8F, 0x90, and 0x9D are formally undefined /// by Windows 1252. These are in XML's RestrictedChar set, so we map them to a /// space. /// <para /> /// The official Latin-1 characters in the range 0xA0..0xFF are converted into /// the Unicode Latin Supplement range U+00A0 - U+00FF. /// <para /> /// <em>Example:</em> If an Euro-symbol (€) appears in the byte buffer (0xE2, 0x82, 0xAC), /// it will be left as is. But if only the first two bytes are appearing, /// followed by an ASCII char a (0xE2 - 0x82 - 0x41), it will be converted to /// 0xC3, 0xA2 (â) - 0xE2, 0x80, 0x9A (‚) - 0x41 (a). /// </remarks> /// <param name="buffer">a byte buffer contain</param> /// <returns>Returns a new buffer containing valid UTF-8</returns> public static ByteBuffer Convert(ByteBuffer buffer) { if (ReferenceEquals(buffer.GetEncoding(), Encoding.UTF8)) { // the buffer containing one UTF-8 char (up to 8 bytes) var readAheadBuffer = new byte[8]; // the number of bytes read ahead. var readAhead = 0; // expected UTF8 bytes to come var expectedBytes = 0; // output buffer with estimated length var output = new ByteBuffer(buffer.Length() * 4 / 3); var state = StateStart; for (var i = 0; i < buffer.Length(); i++) { var b = buffer.CharAt(i); switch (state) { case StateStart: default: { if (b < 0x7F) { output.Append(unchecked((byte)b)); } else { if (b >= 0xC0) { // start of UTF8 sequence expectedBytes = -1; var test = b; for (; expectedBytes < 8 && (test & 0x80) == 0x80; test = test << 1) { expectedBytes++; } readAheadBuffer[readAhead++] = unchecked((byte)b); state = StateUtf8Char; } else { // implicitly: b >= 0x80 && b < 0xC0 // invalid UTF8 start char, assume to be Latin-1 var utf8 = ConvertToUtf8(unchecked((byte)b)); output.Append(utf8); } } break; } case StateUtf8Char: { if (expectedBytes > 0 && (b & 0xC0) == 0x80) { // valid UTF8 char, add to readAheadBuffer readAheadBuffer[readAhead++] = unchecked((byte)b); expectedBytes--; if (expectedBytes == 0) { output.Append(readAheadBuffer, 0, readAhead); readAhead = 0; state = StateStart; } } else { // invalid UTF8 char: // 1. convert first of seq to UTF8 var utf8 = ConvertToUtf8(readAheadBuffer[0]); output.Append(utf8); // 2. continue processing at second byte of sequence i = i - readAhead; readAhead = 0; state = StateStart; } break; } } } // loop ends with "half" Utf8 char --> assume that the bytes are Latin-1 if (state == StateUtf8Char) { for (var j = 0; j < readAhead; j++) { var b = readAheadBuffer[j]; var utf8 = ConvertToUtf8(b); output.Append(utf8); } } return output; } // Latin-1 fixing applies only to UTF-8 return buffer; }