private const int MAXIMUM_LENGTH = 1 * 10 * 1024 * 1024; // 10MB #endregion #region utilities private void TestConformance(TimedMethod[] compressors, TimedMethod[] decompressors) { var provider = new BlockDataProvider(Utilities.GetSilesiaCorpusFolder()); var r = new Random(0); Console.WriteLine("Architecture: {0}bit", IntPtr.Size * 8); var total = 0; const long limit = 1L * 1024 * 1024 * 1024; var last_pct = 0; while (total < limit) { var length = Utilities.RandomLength(r, MAXIMUM_LENGTH); var block = provider.GetBytes(length); TestData(block, compressors, decompressors); total += block.Length; var pct = (int)((double)total * 100 / limit); if (pct > last_pct) { Console.WriteLine("{0}%...", pct); last_pct = pct; } } }
private static void Warmup(TimedMethod compressor, TimedMethod decompressor) { Console.WriteLine("Warming up..."); const int length = 1 * 1024 * 1024; var data = new byte[length]; var gen = new Random(0); gen.NextBytes(data); var compressed = compressor.Warmup(data, length); decompressor.Warmup(compressed, length); }
private static void TestPerformance(TimedMethod[] compressors, TimedMethod[] decompressors) { var names = compressors.Select(c => c.Name).ToArray(); foreach (var name in names) { var compressor = compressors.First(c => c.Name == name); var decompressor = decompressors.First(d => d.Name == name); Console.WriteLine("---- {0} ----", name); Warmup(compressor, decompressor); var provider = new FileDataProvider(Utilities.GetSilesiaCorpusFolder()); long total = 0; const long limit = 1L*1024*1024*1024; var last_pct = 0; while (total < limit) { var block = provider.GetBytes(); TestSpeed(block, compressor, decompressor); total += block.Length; var pct = (int)((double)total*100/limit); if (pct > last_pct) { Console.WriteLine("{0}%...", pct); last_pct = pct; } } GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); Thread.Sleep(1000); } Console.WriteLine("---- Results ----"); Console.WriteLine("Architecture: {0}bit", IntPtr.Size*8); Console.WriteLine("Compression:"); foreach (var compressor in compressors) { Console.WriteLine(" {0}: {1:0.00}MB/s ({2:0.00}%)", compressor.Name, compressor.Speed, compressor.Ratio); } Console.WriteLine("Decompression:"); foreach (var decompressor in decompressors) { Console.WriteLine(" {0}: {1:0.00}MB/s", decompressor.Name, decompressor.Speed); } }
private static void Warmup(TimedMethod compressor, TimedMethod decompressor) { const int length = 1 * 1024 * 1024; var data = new byte[length]; var gen = new Random(0); gen.NextBytes(data); var compressed = compressor.Warmup(data, length); AssertEqual(data, decompressor.Warmup(compressed, length), compressor.Name); }
private static void TestSpeed(byte[] original, TimedMethod compressor, TimedMethod decompressor) { int length = original.Length; byte[] compressed = compressor.Run(original, length); AssertEqual(original, decompressor.Run(compressed, length), compressor.Name); }
private static void UpdateResults(string resultsFile, string codecCode, TimedMethod compressor, TimedMethod decompressor) { Console.WriteLine("Updating results"); var serializer = new XmlSerializer(typeof(Results)); Results results; if (File.Exists(resultsFile)) { using (var stream = File.OpenRead(resultsFile)) results = (Results)serializer.Deserialize(stream); } else { results = new Results(); } if (results.Update(codecCode, compressor.Speed, decompressor.Speed, compressor.InputLength, compressor.OutputLength)) { Console.WriteLine("IMPROVED!!!"); } using (var stream = File.Create(resultsFile)) serializer.Serialize(stream, results); results.SaveAsCSV(resultsFile + ".csv"); Console.WriteLine("---- Latest results ----"); foreach (var item in results.Items) { Console.WriteLine(" {0}: {1:0.00} / {2:0.00}", item.Codec, item.CompressionSpeed, item.DecompressionSpeed); } Console.WriteLine("------------------------"); }
private static string SelectCodec( string codec, string architecture, out TimedMethod compressor, out TimedMethod decompressor) { string codecCode; switch (codec.ToLower()) { // compare implementation for LZ4 only case "mixedmode32": codecCode = string.Format("MixedMode32@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode32(b, 0, b.Length, l)); break; case "mixedmode64": codecCode = string.Format("MixedMode64@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode64(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode64(b, 0, b.Length, l)); break; case "cppcli32": codecCode = string.Format("C++/CLI32@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Decode32(b, 0, b.Length, l)); break; case "cppcli64": codecCode = string.Format("C++/CLI64@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Encode64(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Decode64(b, 0, b.Length, l)); break; case "unsafe64": codecCode = string.Format("Unsafe64@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode64(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode64(b, 0, b.Length, l)); break; case "unsafe32": codecCode = string.Format("Unsafe32@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode32(b, 0, b.Length, l)); break; case "safe64": codecCode = string.Format("Safe64@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode64(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l)); break; case "safe32": codecCode = string.Format("Safe32@{0}", architecture); compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode32(b, 0, b.Length, l)); break; // compare different algorithms (LZ4, LZO, Snappy, QuickLZ) case "lz4.native": codecCode = string.Format("LZ4.native@{0}", architecture); if (IntPtr.Size == 4) { compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode32(b, 0, b.Length, l)); } else { compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode64(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode64(b, 0, b.Length, l)); } break; case "lz4.unsafe": codecCode = string.Format("LZ4.unsafe@{0}", architecture); if (IntPtr.Size == 4) { compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode64(b, 0, b.Length, l)); } else { compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode64(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode32(b, 0, b.Length, l)); } break; #if !X64 // LZO is available only in 32-bit mode case "lzo1x.native": codecCode = string.Format("LZO1X.native@{0}", architecture); compressor = new TimedMethod(codecCode, NativeLZO1XCompressor); decompressor = new TimedMethod(codecCode, NativeLZODecompressor); break; case "lzo1x11.native": codecCode = string.Format("LZO1X11.native@{0}", architecture); compressor = new TimedMethod(codecCode, NativeLZO1X11Compressor); decompressor = new TimedMethod(codecCode, NativeLZODecompressor); break; case "lzo1x12.native": codecCode = string.Format("LZO1X12.native@{0}", architecture); compressor = new TimedMethod(codecCode, NativeLZO1X12Compressor); decompressor = new TimedMethod(codecCode, NativeLZODecompressor); break; case "lzo1x15.native": codecCode = string.Format("LZO1X15.native@{0}", architecture); compressor = new TimedMethod(codecCode, NativeLZO1X15Compressor); decompressor = new TimedMethod(codecCode, NativeLZODecompressor); break; case "lzo1x999.native": codecCode = string.Format("LZO1X999.native@{0}", architecture); compressor = new TimedMethod(codecCode, NativeLZO1X999Compressor); decompressor = new TimedMethod(codecCode, NativeLZODecompressor); break; #endif case "lz4sharp.unsafe": codecCode = string.Format("LZ4Sharp.unsafe@{0}", architecture); if (IntPtr.Size == 4) { var lz4sharp_compressor32 = new LZ4Sharp.LZ4Compressor32(); var lz4sharp_decompressor32 = new LZ4Sharp.LZ4Decompressor32(); compressor = new TimedMethod(codecCode, (b, l) => lz4sharp_compressor32.Compress(b)); decompressor = new TimedMethod(codecCode, (b, l) => { var output = new byte[l]; lz4sharp_decompressor32.DecompressKnownSize(b, output); return output; }); } else { var lz4sharp_compressor64 = new LZ4Sharp.LZ4Compressor64(); var lz4sharp_decompressor64 = new LZ4Sharp.LZ4Decompressor64(); compressor = new TimedMethod(codecCode, (b, l) => lz4sharp_compressor64.Compress(b)); decompressor = new TimedMethod(codecCode, (b, l) => { var output = new byte[l]; lz4sharp_decompressor64.DecompressKnownSize(b, output); return output; }); } break; case "quicklz.native": codecCode = string.Format("QuickLZ.native@{0}", architecture); compressor = new TimedMethod(codecCode, NativeQuickLZCompressor); decompressor = new TimedMethod(codecCode, NativeQuickLZDecompressor); break; case "snappy.native": codecCode = string.Format("Snappy.native@{0}", architecture); compressor = new TimedMethod(codecCode, NativeSnappyCompressor); decompressor = new TimedMethod(codecCode, NativeSnappyDecompressor); break; // safe compressors case "lz4.safe": codecCode = string.Format("LZ4.safe@{0}", architecture); if (IntPtr.Size == 4) { compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l)); } else { compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l)); decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l)); } break; case "lzf.safe": codecCode = string.Format("LZF.safe@{0}", architecture); compressor = new TimedMethod(codecCode, SafeLZFCompressor); decompressor = new TimedMethod(codecCode, SafeLZFDecompressor); break; case "quicklz1.safe": codecCode = string.Format("QuickLZ1.safe@{0}", architecture); compressor = new TimedMethod(codecCode, SafeQuickLZ1Compressor); decompressor = new TimedMethod(codecCode, SafeQuickLZDecompressor); break; case "quicklz3.safe": codecCode = string.Format("QuickLZ3.safe@{0}", architecture); compressor = new TimedMethod(codecCode, SafeQuickLZ3Compressor); decompressor = new TimedMethod(codecCode, SafeQuickLZDecompressor); break; case "deflate.safe": codecCode = string.Format("Deflate.safe@{0}", architecture); compressor = new TimedMethod(codecCode, DeflateCompressor); decompressor = new TimedMethod(codecCode, DeflateDecompressor); break; default: throw new ArgumentException(string.Format("Unknown codec: {0}", codec)); } return codecCode; }
private void TestConformance(TimedMethod[] compressors, TimedMethod[] decompressors) { var provider = new BlockDataProvider(Utilities.TEST_DATA_FOLDER); var r = new Random(0); Console.WriteLine("Architecture: {0}bit", IntPtr.Size * 8); var total = 0; const long limit = 1L * 1024 * 1024 * 1024; var last_pct = 0; while (total < limit) { var length = Utilities.RandomLength(r, MAXIMUM_LENGTH); var block = provider.GetBytes(length); TestData(block, compressors, decompressors); total += block.Length; var pct = (int)((double)total * 100 / limit); if (pct > last_pct) { Console.WriteLine("{0}%...", pct); last_pct = pct; } } /* The performance results from this test are completely unreliable Too much garbage collection and caching. So, no need to mislead anyone. Console.WriteLine("Compression:"); foreach (var compressor in compressors) { Console.WriteLine(" {0}: {1:0.00}MB/s", compressor.Name, compressor.Speed); } Console.WriteLine("Decompression:"); foreach (var decompressor in decompressors) { Console.WriteLine(" {0}: {1:0.00}MB/s", decompressor.Name, decompressor.Speed); } */ }