public void TestBasic1()
        {
            var freqs   = new uint[] { 10, 30, 10 }; // Symbol 0 occurs 10x, symbol 1 occurs 30x, symbol 2 occurs 10x
            var ms      = new MemoryStream();
            var context = new ArithmeticSymbolArrayContext(freqs);
            var encoder = new ArithmeticCodingWriter(ms, context);

            for (int i = 0; i < 10; i++)
            {
                encoder.WriteSymbol(1);
                encoder.WriteSymbol(0);
                encoder.WriteSymbol(1);
                encoder.WriteSymbol(2);
                encoder.WriteSymbol(1);
            }
            encoder.Finalize();
            ms.WriteByte(47);

            ms = new MemoryStream(ms.ToArray());
            var decoder = new ArithmeticCodingReader(ms, context);

            for (int i = 0; i < 10; i++)
            {
                Assert.AreEqual(1, decoder.ReadSymbol());
                Assert.AreEqual(0, decoder.ReadSymbol());
                Assert.AreEqual(1, decoder.ReadSymbol());
                Assert.AreEqual(2, decoder.ReadSymbol());
                Assert.AreEqual(1, decoder.ReadSymbol());
            }
            decoder.Finalize();
            Assert.AreEqual(47, ms.ReadByte());
        }
        private static void testExtremeProbs(uint[] freqs, int[] symbols)
        {
            var ms      = new MemoryStream();
            var encoder = new ArithmeticCodingWriter(ms, freqs);

            foreach (var symbol in symbols)
            {
                encoder.WriteSymbol(symbol);
            }
            encoder.Finalize();
            var expectedEnding = ms.Position;

            ms.WriteByte(47);
            var bytes = ms.ToArray();

            ms = new MemoryStream(bytes);
            var decoder = new ArithmeticCodingReader(ms, freqs);

            for (int i = 0; i < symbols.Length; i++)
            {
                Assert.AreEqual(symbols[i], decoder.ReadSymbol());
            }
            decoder.Finalize();
            Assert.AreEqual(expectedEnding, ms.Position);
            Assert.AreEqual(47, ms.ReadByte());
        }
        public void TestNormalByteSequence()
        {
            // The encoding of bytes under a 0..255 context with equal probability should end up outputting those bytes unchanged
            var ms      = new MemoryStream();
            var encoder = new ArithmeticCodingWriter(ms, new ArithmeticSymbolArrayContext(256));

            for (int i = 0; i <= 255; i++)
            {
                encoder.WriteSymbol(i);
            }
            encoder.Finalize(false);
            var result = ms.ToArray();

            for (int i = 0; i <= 255; i++)
            {
                Assert.AreEqual(i, result[i]);
            }

            ms = new MemoryStream(result);
            var decoder = new ArithmeticCodingReader(ms, new ArithmeticSymbolArrayContext(256));

            for (int i = 0; i <= 255; i++)
            {
                Assert.AreEqual(i, decoder.ReadSymbol());
            }
            decoder.Finalize(false);
        }
        public void TestRandom()
        {
            var rnd = new Random(468);

            for (int t = 0; t < 2000; t++)
            {
                var freqs   = newArray(rnd.Next(1, 300), _ => (uint)rnd.Next(1, 500));
                var symbols = newArray(rnd.Next(0, 1000), _ => (uint)rnd.Next(0, freqs.Length)).Select(e => (int)e).ToArray();
                var ms      = new MemoryStream();
                var encoder = new ArithmeticCodingWriter(ms, freqs);
                for (int i = 0; i < symbols.Length; i++)
                {
                    encoder.WriteSymbol(symbols[i]);
                }
                encoder.Finalize();
                var expectedEnding = ms.Position;
                ms.WriteByte(47);
                var bytes = ms.ToArray();

                ms = new MemoryStream(bytes);
                var decoder = new ArithmeticCodingReader(ms, freqs);
                for (int i = 0; i < symbols.Length; i++)
                {
                    Assert.AreEqual(symbols[i], decoder.ReadSymbol());
                }
                decoder.Finalize();
                Assert.AreEqual(expectedEnding, ms.Position);
                Assert.AreEqual(47, ms.ReadByte());
            }
        }
        private void testBasic(int length)
        {
            var freqs   = newArray(256, v => 256 - (uint)v);
            var ms      = new MemoryStream();
            var encoder = new ArithmeticCodingWriter(ms, freqs);

            for (int i = 0; i < length; i++)
            {
                encoder.WriteSymbol(i % 256);
            }
            encoder.Finalize(false);
            var expectedEnding = ms.Position;

            ms.Write(new byte[32], 0, 32);
            var bytes = ms.ToArray();

            ms = new MemoryStream(bytes);
            var decoder = new ArithmeticCodingReader(ms, freqs);

            for (int i = 0; i < length; i++)
            {
                var sym = decoder.ReadSymbol();
                Assert.AreEqual(i % 256, sym);
            }
            decoder.Finalize(false);
            Assert.AreEqual(expectedEnding, ms.Position);
        }
        public void TestSingleSymbol()
        {
            var freqs   = new uint[] { 1 };
            var ms      = new MemoryStream();
            var encoder = new ArithmeticCodingWriter(ms, freqs);

            for (int i = 0; i < 100; i++)
            {
                encoder.WriteSymbol(0);
            }
            encoder.Finalize(false);
            Assert.AreEqual(5, ms.Position);

            ms = new MemoryStream(ms.ToArray());
            var decoder = new ArithmeticCodingReader(ms, freqs);

            for (int i = 0; i < 100; i++)
            {
                Assert.AreEqual(0, decoder.ReadSymbol());
            }
            decoder.Finalize(false);
            Assert.AreEqual(5, ms.Position);
        }
        public void TestExtremeProbabilities1()
        {
            // This test encodes and decodes a sequence of N 1's, followed by a single 0, where the frequency of 1 is extremely high and the frequency of 0 is 1 (minimal).
            // For very large frequencies, the encoder overflows after a certain number of 1's.
            // This test verifies correct operation for all sequences of length 1..1000, then goes up in bigger increments until a sequence of ~10 million 1's.
            // A 100 billion long sequence has been tested manually but takes far too long for a unit test (encoded: FF FF FF FF FF FD 21 DB A1 79 + sync padding)

            // Maximum frequency vs first failure at 1's count:
            // 0xFFFF_FFFE: 2
            // 0xFFFF_FFF0: 16
            // 0xFFFF_FF00: 256
            // 0xFFFF_F000: 4096
            // 0xFFFF_0000: 65536
            // 0xF000_0000: 268,435,455
            // 0x8000_0001: 2,147,483,647
            // 0x8000_0000: correct to at least 2.2 billion
            // 0x7FFF_FFFF: correct to at least 100 billion

            var freqs = new[] { 1u, ArithmeticSymbolContext.MaxTotal - 1 };
            int count = 0;

            while (true)
            {
                var ms      = new MemoryStream();
                var encoder = new ArithmeticCodingWriter(ms, freqs);
                for (int i = 0; i < count; i++)
                {
                    encoder.WriteSymbol(1);
                }
                encoder.WriteSymbol(0);
                encoder.Finalize();
                var expectedEnding = ms.Position;
                ms.WriteByte(47);
                var bytes = ms.ToArray();

                ms = new MemoryStream(bytes);
                var decoder = new ArithmeticCodingReader(ms, freqs);
                for (int i = 0; i < count; i++)
                {
                    Assert.AreEqual(1, decoder.ReadSymbol());
                }
                Assert.AreEqual(0, decoder.ReadSymbol());
                decoder.Finalize();
                Assert.AreEqual(expectedEnding, ms.Position);
                Assert.AreEqual(47, ms.ReadByte());

                if (count < 1000)
                {
                    count++;
                }
                else if (count < 10_000)
                {
                    count += 997;
                }
                else if (count < 100_000)
                {
                    count = 9_999_991;
                }
                else
                {
                    break;
                }
            }
        }
        public void TestAdvanced()
        {
            _rnd = new Random(12345);
            int max     = 1000;
            var symbols = Enumerable.Range(1, 100_000).Select(_ => _rnd.Next(0, max)).ToArray();

            var mainContext      = new ArithmeticSymbolArrayContext(max, _ => 1);
            var secondaryContext = new ArithmeticSymbolArrayContext(new uint[] { 3, 2, 1 });

            var ms      = new MemoryStream();
            var encoder = new ArithmeticCodingWriter(ms, mainContext);

            writeInt(ms, 12345);
            for (int i = 0; i < symbols.Length; i++)
            {
                encoder.WriteSymbol(symbols[i]);
                mainContext.IncrementSymbolFrequency(symbols[i]);
                encoder.SetContext(mainContext);
                if (i % 1000 == 999)
                {
                    encoder.SetContext(secondaryContext);
                    encoder.WriteSymbol(0);
                    encoder.WriteSymbol(1);
                    encoder.WriteSymbol(0);
                    encoder.WriteSymbol(1);
                    encoder.WriteSymbol(0);
                    encoder.WriteSymbol(2);
                    encoder.SetContext(mainContext);
                }
            }
            encoder.Finalize(false);
            writeInt(ms, -54321); // to verify that the stream ends where we think it ends
            var encoded = ms.ToArray();


            ms          = new MemoryStream(encoded);
            mainContext = new ArithmeticSymbolArrayContext(max, _ => 1); // reset frequencies
            Assert.AreEqual(12345, readInt(ms));
            var decoder = new ArithmeticCodingReader(ms, mainContext);

            for (int i = 0; i < symbols.Length; i++)
            {
                var sym = decoder.ReadSymbol();
                Assert.AreEqual(symbols[i], sym);
                mainContext.IncrementSymbolFrequency(sym);
                decoder.SetContext(mainContext);
                if (i % 1000 == 999)
                {
                    decoder.SetContext(secondaryContext);
                    Assert.AreEqual(0, decoder.ReadSymbol());
                    Assert.AreEqual(1, decoder.ReadSymbol());
                    Assert.AreEqual(0, decoder.ReadSymbol());
                    Assert.AreEqual(1, decoder.ReadSymbol());
                    Assert.AreEqual(0, decoder.ReadSymbol());
                    Assert.AreEqual(2, decoder.ReadSymbol());
                    decoder.SetContext(mainContext);
                }
            }
            decoder.Finalize(false);
            Assert.AreEqual(-54321, readInt(ms));
        }