Example #1
0
		private const int MAXIMUM_LENGTH = 1 * 10 * 1024 * 1024; // 10MB

		#endregion

		#region utilities

		private void TestConformance(TimedMethod[] compressors, TimedMethod[] decompressors)
		{
			var provider = new BlockDataProvider(Utilities.GetSilesiaCorpusFolder());

			var r = new Random(0);

			Console.WriteLine("Architecture: {0}bit", IntPtr.Size * 8);

			var total = 0;
			const long limit = 1L * 1024 * 1024 * 1024;
			var last_pct = 0;

			while (total < limit)
			{
				var length = Utilities.RandomLength(r, MAXIMUM_LENGTH);
				var block = provider.GetBytes(length);
				TestData(block, compressors, decompressors);
				total += block.Length;
				var pct = (int)((double)total * 100 / limit);
				if (pct > last_pct)
				{
					Console.WriteLine("{0}%...", pct);
					last_pct = pct;
				}
			}
		}
Example #2
0
		private static void Warmup(TimedMethod compressor, TimedMethod decompressor)
		{
			Console.WriteLine("Warming up...");

			const int length = 1 * 1024 * 1024;
			var data = new byte[length];
			var gen = new Random(0);
			gen.NextBytes(data);
			var compressed = compressor.Warmup(data, length);
			decompressor.Warmup(compressed, length);
		}
Example #3
0
		private static void TestPerformance(TimedMethod[] compressors, TimedMethod[] decompressors)
		{
			var names = compressors.Select(c => c.Name).ToArray();

			foreach (var name in names)
			{
				var compressor = compressors.First(c => c.Name == name);
				var decompressor = decompressors.First(d => d.Name == name);

				Console.WriteLine("---- {0} ----", name);

				Warmup(compressor, decompressor);

				var provider = new FileDataProvider(Utilities.GetSilesiaCorpusFolder());
				long total = 0;
				const long limit = 1L*1024*1024*1024;
				var last_pct = 0;

				while (total < limit)
				{
					var block = provider.GetBytes();
					TestSpeed(block, compressor, decompressor);
					total += block.Length;
					var pct = (int)((double)total*100/limit);
					if (pct > last_pct)
					{
						Console.WriteLine("{0}%...", pct);
						last_pct = pct;
					}
				}

				GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
				Thread.Sleep(1000);
			}

			Console.WriteLine("---- Results ----");

			Console.WriteLine("Architecture: {0}bit", IntPtr.Size*8);
			Console.WriteLine("Compression:");
			foreach (var compressor in compressors)
			{
				Console.WriteLine("  {0}: {1:0.00}MB/s ({2:0.00}%)", compressor.Name, compressor.Speed, compressor.Ratio);
			}

			Console.WriteLine("Decompression:");
			foreach (var decompressor in decompressors)
			{
				Console.WriteLine("  {0}: {1:0.00}MB/s", decompressor.Name, decompressor.Speed);
			}
		}
Example #4
0
        private static void Warmup(TimedMethod compressor, TimedMethod decompressor)
        {
            const int length = 1 * 1024 * 1024;
            var data = new byte[length];
            var gen = new Random(0);
            gen.NextBytes(data);

            var compressed = compressor.Warmup(data, length);
            AssertEqual(data, decompressor.Warmup(compressed, length), compressor.Name);
        }
Example #5
0
 private static void TestSpeed(byte[] original, TimedMethod compressor, TimedMethod decompressor)
 {
     int length = original.Length;
     byte[] compressed = compressor.Run(original, length);
     AssertEqual(original, decompressor.Run(compressed, length), compressor.Name);
 }
Example #6
0
        private static void UpdateResults(string resultsFile, string codecCode, TimedMethod compressor, TimedMethod decompressor)
        {
            Console.WriteLine("Updating results");

            var serializer = new XmlSerializer(typeof(Results));
            Results results;

            if (File.Exists(resultsFile))
            {
                using (var stream = File.OpenRead(resultsFile)) results = (Results)serializer.Deserialize(stream);
            }
            else
            {
                results = new Results();
            }

            if (results.Update(codecCode, compressor.Speed, decompressor.Speed, compressor.InputLength, compressor.OutputLength))
            {
                Console.WriteLine("IMPROVED!!!");
            }

            using (var stream = File.Create(resultsFile)) serializer.Serialize(stream, results);
            results.SaveAsCSV(resultsFile + ".csv");

            Console.WriteLine("---- Latest results ----");
            foreach (var item in results.Items)
            {
                Console.WriteLine("  {0}: {1:0.00} / {2:0.00}", item.Codec, item.CompressionSpeed, item.DecompressionSpeed);
            }
            Console.WriteLine("------------------------");
        }
Example #7
0
        private static string SelectCodec(
			string codec, string architecture, 
			out TimedMethod compressor, 
			out TimedMethod decompressor)
        {
            string codecCode;
            switch (codec.ToLower())
            {
                // compare implementation for LZ4 only

                case "mixedmode32":
                    codecCode = string.Format("MixedMode32@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode32(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode32(b, 0, b.Length, l));
                    break;
                case "mixedmode64":
                    codecCode = string.Format("MixedMode64@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode64(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode64(b, 0, b.Length, l));
                    break;
                case "cppcli32":
                    codecCode = string.Format("C++/CLI32@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Encode32(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Decode32(b, 0, b.Length, l));
                    break;
                case "cppcli64":
                    codecCode = string.Format("C++/CLI64@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Encode64(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Decode64(b, 0, b.Length, l));
                    break;
                case "unsafe64":
                    codecCode = string.Format("Unsafe64@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode64(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode64(b, 0, b.Length, l));
                    break;
                case "unsafe32":
                    codecCode = string.Format("Unsafe32@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode32(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode32(b, 0, b.Length, l));
                    break;
                case "safe64":
                    codecCode = string.Format("Safe64@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode64(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l));
                    break;
                case "safe32":
                    codecCode = string.Format("Safe32@{0}", architecture);
                    compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l));
                    decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode32(b, 0, b.Length, l));
                    break;

                // compare different algorithms (LZ4, LZO, Snappy, QuickLZ)

                case "lz4.native":
                    codecCode = string.Format("LZ4.native@{0}", architecture);
                    if (IntPtr.Size == 4)
                    {
                        compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode32(b, 0, l));
                        decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode32(b, 0, b.Length, l));
                    }
                    else
                    {
                        compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode64(b, 0, l));
                        decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode64(b, 0, b.Length, l));
                    }
                    break;

                case "lz4.unsafe":
                    codecCode = string.Format("LZ4.unsafe@{0}", architecture);
                    if (IntPtr.Size == 4)
                    {
                        compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode32(b, 0, l));
                        decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode64(b, 0, b.Length, l));
                    }
                    else
                    {
                        compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode64(b, 0, l));
                        decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode32(b, 0, b.Length, l));
                    }
                    break;

            #if !X64
                // LZO is available only in 32-bit mode

                case "lzo1x.native":
                    codecCode = string.Format("LZO1X.native@{0}", architecture);
                    compressor = new TimedMethod(codecCode, NativeLZO1XCompressor);
                    decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
                    break;

                case "lzo1x11.native":
                    codecCode = string.Format("LZO1X11.native@{0}", architecture);
                    compressor = new TimedMethod(codecCode, NativeLZO1X11Compressor);
                    decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
                    break;

                case "lzo1x12.native":
                    codecCode = string.Format("LZO1X12.native@{0}", architecture);
                    compressor = new TimedMethod(codecCode, NativeLZO1X12Compressor);
                    decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
                    break;

                case "lzo1x15.native":
                    codecCode = string.Format("LZO1X15.native@{0}", architecture);
                    compressor = new TimedMethod(codecCode, NativeLZO1X15Compressor);
                    decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
                    break;

                case "lzo1x999.native":
                    codecCode = string.Format("LZO1X999.native@{0}", architecture);
                    compressor = new TimedMethod(codecCode, NativeLZO1X999Compressor);
                    decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
                    break;

            #endif

                case "lz4sharp.unsafe":
                    codecCode = string.Format("LZ4Sharp.unsafe@{0}", architecture);
                    if (IntPtr.Size == 4)
                    {
                        var lz4sharp_compressor32 = new LZ4Sharp.LZ4Compressor32();
                        var lz4sharp_decompressor32 = new LZ4Sharp.LZ4Decompressor32();
                        compressor = new TimedMethod(codecCode, (b, l) => lz4sharp_compressor32.Compress(b));
                        decompressor = new TimedMethod(codecCode, (b, l) =>
                            {
                                var output = new byte[l];
                                lz4sharp_decompressor32.DecompressKnownSize(b, output);
                                return output;
                            });
                    }
                    else
                    {
                        var lz4sharp_compressor64 = new LZ4Sharp.LZ4Compressor64();
                        var lz4sharp_decompressor64 = new LZ4Sharp.LZ4Decompressor64();
                        compressor = new TimedMethod(codecCode, (b, l) => lz4sharp_compressor64.Compress(b));
                        decompressor = new TimedMethod(codecCode, (b, l) =>
                            {
                                var output = new byte[l];
                                lz4sharp_decompressor64.DecompressKnownSize(b, output);
                                return output;
                            });
                    }
                    break;

                case "quicklz.native":
                    codecCode = string.Format("QuickLZ.native@{0}", architecture);
                    compressor = new TimedMethod(codecCode, NativeQuickLZCompressor);
                    decompressor = new TimedMethod(codecCode, NativeQuickLZDecompressor);
                    break;

                case "snappy.native":
                    codecCode = string.Format("Snappy.native@{0}", architecture);
                    compressor = new TimedMethod(codecCode, NativeSnappyCompressor);
                    decompressor = new TimedMethod(codecCode, NativeSnappyDecompressor);
                    break;

                // safe compressors

                case "lz4.safe":
                    codecCode = string.Format("LZ4.safe@{0}", architecture);
                    if (IntPtr.Size == 4)
                    {
                        compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l));
                        decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l));
                    }
                    else
                    {
                        compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l));
                        decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l));
                    }
                    break;

                case "lzf.safe":
                    codecCode = string.Format("LZF.safe@{0}", architecture);
                    compressor = new TimedMethod(codecCode, SafeLZFCompressor);
                    decompressor = new TimedMethod(codecCode, SafeLZFDecompressor);
                    break;

                case "quicklz1.safe":
                    codecCode = string.Format("QuickLZ1.safe@{0}", architecture);
                    compressor = new TimedMethod(codecCode, SafeQuickLZ1Compressor);
                    decompressor = new TimedMethod(codecCode, SafeQuickLZDecompressor);
                    break;

                case "quicklz3.safe":
                    codecCode = string.Format("QuickLZ3.safe@{0}", architecture);
                    compressor = new TimedMethod(codecCode, SafeQuickLZ3Compressor);
                    decompressor = new TimedMethod(codecCode, SafeQuickLZDecompressor);
                    break;

                case "deflate.safe":
                    codecCode = string.Format("Deflate.safe@{0}", architecture);
                    compressor = new TimedMethod(codecCode, DeflateCompressor);
                    decompressor = new TimedMethod(codecCode, DeflateDecompressor);
                    break;

                default:
                    throw new ArgumentException(string.Format("Unknown codec: {0}", codec));
            }
            return codecCode;
        }
Example #8
0
        private void TestConformance(TimedMethod[] compressors, TimedMethod[] decompressors)
        {
            var provider = new BlockDataProvider(Utilities.TEST_DATA_FOLDER);

            var r = new Random(0);

            Console.WriteLine("Architecture: {0}bit", IntPtr.Size * 8);

            var total = 0;
            const long limit = 1L * 1024 * 1024 * 1024;
            var last_pct = 0;

            while (total < limit)
            {
                var length = Utilities.RandomLength(r, MAXIMUM_LENGTH);
                var block = provider.GetBytes(length);
                TestData(block, compressors, decompressors);
                total += block.Length;
                var pct = (int)((double)total * 100 / limit);
                if (pct > last_pct)
                {
                    Console.WriteLine("{0}%...", pct);
                    last_pct = pct;
                }
            }

            /*

            The performance results from this test are completely unreliable
            Too much garbage collection and caching.
            So, no need to mislead anyone.

            Console.WriteLine("Compression:");
            foreach (var compressor in compressors)
            {
                Console.WriteLine("  {0}: {1:0.00}MB/s", compressor.Name, compressor.Speed);
            }

            Console.WriteLine("Decompression:");
            foreach (var decompressor in decompressors)
            {
                Console.WriteLine("  {0}: {1:0.00}MB/s", decompressor.Name, decompressor.Speed);
            }
            */
        }