public void TestMultithreadedDecoding()
{
const string expectedText =
"\r\n" +
"class Program\r\n" +
"{\r\n" +
" static void Main()\r\n" +
" {\r\n" +
" string s = \"class C { \u0410\u0411\u0412 x; }\";\r\n" +
" foreach (char ch in s) System.Console.WriteLine(\"{0:x2}\", (int)ch);\r\n" +
" }\r\n" +
"}\r\n";
var encoding = new UTF8Encoding(false);
string path = Temp.CreateFile().WriteAllBytes(encoding.GetBytes(expectedText)).Path;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
};
Parallel.For(0, 500, parallelOptions, i =>
{
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read))
{
string actualText;
actualText = EncodedStringText.DetectEncodingAndDecode(stream);
Assert.Equal(expectedText, actualText);
}
});
}
public void Decode_Utf8()
{
var utf8 = new UTF8Encoding(false, true);
var text = "abc def baz aeiouy äëïöüû";
var bytes = GetBytes(utf8, text);
// Detect encoding should correctly pick UTF-8
using (var stream = new MemoryStream(bytes))
{
Assert.Equal(text, EncodedStringText.DetectEncodingAndDecode(stream));
Assert.True(stream.CanRead);
}
}
public void Decode_NonUtf8()
{
var utf8 = new UTF8Encoding(false, true);
var text = "abc def baz aeiouy " + Encoding.Default.GetString(new byte[] { 0x80, 0x92, 0xA4, 0xB6, 0xC9, 0xDB, 0xED, 0xFF });
var bytes = GetBytes(Encoding.Default, text);
// Encoding.Default should not decode to UTF-8
using (var stream = new MemoryStream(bytes))
{
Assert.Throws(typeof(DecoderFallbackException), () => EncodedStringText.Decode(stream, utf8));
Assert.True(stream.CanRead);
}
// Detect encoding should correctly pick Encoding.Default
using (var stream = new MemoryStream(bytes))
{
Assert.Equal(text, EncodedStringText.DetectEncodingAndDecode(stream));
Assert.True(stream.CanRead);
}
}
public void Decode_NonUtf8()
{
var encoding1252 = Encoding.GetEncoding(1252);
var utf8 = new UTF8Encoding(false, true);
var text = "abc def baz aeiouy äëïöüû";
var bytes = GetBytes(encoding1252, text);
// 1252 should not decode to UTF-8
using (var stream = new MemoryStream(bytes))
{
Assert.Throws(typeof(DecoderFallbackException), () => EncodedStringText.Decode(stream, utf8));
Assert.True(stream.CanRead);
}
// Detect encoding should correctly pick 1252
using (var stream = new MemoryStream(bytes))
{
Assert.Equal(text, EncodedStringText.DetectEncodingAndDecode(stream));
Assert.True(stream.CanRead);
}
}