private const int MAXIMUM_LENGTH = 1 * 10 * 1024 * 1024; // 10MB
#endregion
#region utilities
private void TestConformance(TimedMethod[] compressors, TimedMethod[] decompressors)
{
var provider = new BlockDataProvider(Utilities.GetSilesiaCorpusFolder());
var r = new Random(0);
Console.WriteLine("Architecture: {0}bit", IntPtr.Size * 8);
var total = 0;
const long limit = 1L * 1024 * 1024 * 1024;
var last_pct = 0;
while (total < limit)
{
var length = Utilities.RandomLength(r, MAXIMUM_LENGTH);
var block = provider.GetBytes(length);
TestData(block, compressors, decompressors);
total += block.Length;
var pct = (int)((double)total * 100 / limit);
if (pct > last_pct)
{
Console.WriteLine("{0}%...", pct);
last_pct = pct;
}
}
}
private static void Warmup(TimedMethod compressor, TimedMethod decompressor)
{
Console.WriteLine("Warming up...");
const int length = 1 * 1024 * 1024;
var data = new byte[length];
var gen = new Random(0);
gen.NextBytes(data);
var compressed = compressor.Warmup(data, length);
decompressor.Warmup(compressed, length);
}
private static void TestPerformance(TimedMethod[] compressors, TimedMethod[] decompressors)
{
var names = compressors.Select(c => c.Name).ToArray();
foreach (var name in names)
{
var compressor = compressors.First(c => c.Name == name);
var decompressor = decompressors.First(d => d.Name == name);
Console.WriteLine("---- {0} ----", name);
Warmup(compressor, decompressor);
var provider = new FileDataProvider(Utilities.GetSilesiaCorpusFolder());
long total = 0;
const long limit = 1L*1024*1024*1024;
var last_pct = 0;
while (total < limit)
{
var block = provider.GetBytes();
TestSpeed(block, compressor, decompressor);
total += block.Length;
var pct = (int)((double)total*100/limit);
if (pct > last_pct)
{
Console.WriteLine("{0}%...", pct);
last_pct = pct;
}
}
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
Thread.Sleep(1000);
}
Console.WriteLine("---- Results ----");
Console.WriteLine("Architecture: {0}bit", IntPtr.Size*8);
Console.WriteLine("Compression:");
foreach (var compressor in compressors)
{
Console.WriteLine(" {0}: {1:0.00}MB/s ({2:0.00}%)", compressor.Name, compressor.Speed, compressor.Ratio);
}
Console.WriteLine("Decompression:");
foreach (var decompressor in decompressors)
{
Console.WriteLine(" {0}: {1:0.00}MB/s", decompressor.Name, decompressor.Speed);
}
}
private static void Warmup(TimedMethod compressor, TimedMethod decompressor)
{
const int length = 1 * 1024 * 1024;
var data = new byte[length];
var gen = new Random(0);
gen.NextBytes(data);
var compressed = compressor.Warmup(data, length);
AssertEqual(data, decompressor.Warmup(compressed, length), compressor.Name);
}
private static void TestSpeed(byte[] original, TimedMethod compressor, TimedMethod decompressor)
{
int length = original.Length;
byte[] compressed = compressor.Run(original, length);
AssertEqual(original, decompressor.Run(compressed, length), compressor.Name);
}
private static void UpdateResults(string resultsFile, string codecCode, TimedMethod compressor, TimedMethod decompressor)
{
Console.WriteLine("Updating results");
var serializer = new XmlSerializer(typeof(Results));
Results results;
if (File.Exists(resultsFile))
{
using (var stream = File.OpenRead(resultsFile)) results = (Results)serializer.Deserialize(stream);
}
else
{
results = new Results();
}
if (results.Update(codecCode, compressor.Speed, decompressor.Speed, compressor.InputLength, compressor.OutputLength))
{
Console.WriteLine("IMPROVED!!!");
}
using (var stream = File.Create(resultsFile)) serializer.Serialize(stream, results);
results.SaveAsCSV(resultsFile + ".csv");
Console.WriteLine("---- Latest results ----");
foreach (var item in results.Items)
{
Console.WriteLine(" {0}: {1:0.00} / {2:0.00}", item.Codec, item.CompressionSpeed, item.DecompressionSpeed);
}
Console.WriteLine("------------------------");
}
private static string SelectCodec(
string codec, string architecture,
out TimedMethod compressor,
out TimedMethod decompressor)
{
string codecCode;
switch (codec.ToLower())
{
// compare implementation for LZ4 only
case "mixedmode32":
codecCode = string.Format("MixedMode32@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode32(b, 0, b.Length, l));
break;
case "mixedmode64":
codecCode = string.Format("MixedMode64@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode64(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode64(b, 0, b.Length, l));
break;
case "cppcli32":
codecCode = string.Format("C++/CLI32@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Decode32(b, 0, b.Length, l));
break;
case "cppcli64":
codecCode = string.Format("C++/CLI64@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Encode64(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4cc.LZ4Codec.Decode64(b, 0, b.Length, l));
break;
case "unsafe64":
codecCode = string.Format("Unsafe64@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode64(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode64(b, 0, b.Length, l));
break;
case "unsafe32":
codecCode = string.Format("Unsafe32@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode32(b, 0, b.Length, l));
break;
case "safe64":
codecCode = string.Format("Safe64@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode64(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l));
break;
case "safe32":
codecCode = string.Format("Safe32@{0}", architecture);
compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode32(b, 0, b.Length, l));
break;
// compare different algorithms (LZ4, LZO, Snappy, QuickLZ)
case "lz4.native":
codecCode = string.Format("LZ4.native@{0}", architecture);
if (IntPtr.Size == 4)
{
compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode32(b, 0, b.Length, l));
}
else
{
compressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Encode64(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4mm.LZ4Codec.Decode64(b, 0, b.Length, l));
}
break;
case "lz4.unsafe":
codecCode = string.Format("LZ4.unsafe@{0}", architecture);
if (IntPtr.Size == 4)
{
compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode64(b, 0, b.Length, l));
}
else
{
compressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Encode64(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4n.LZ4Codec.Decode32(b, 0, b.Length, l));
}
break;
#if !X64
// LZO is available only in 32-bit mode
case "lzo1x.native":
codecCode = string.Format("LZO1X.native@{0}", architecture);
compressor = new TimedMethod(codecCode, NativeLZO1XCompressor);
decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
break;
case "lzo1x11.native":
codecCode = string.Format("LZO1X11.native@{0}", architecture);
compressor = new TimedMethod(codecCode, NativeLZO1X11Compressor);
decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
break;
case "lzo1x12.native":
codecCode = string.Format("LZO1X12.native@{0}", architecture);
compressor = new TimedMethod(codecCode, NativeLZO1X12Compressor);
decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
break;
case "lzo1x15.native":
codecCode = string.Format("LZO1X15.native@{0}", architecture);
compressor = new TimedMethod(codecCode, NativeLZO1X15Compressor);
decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
break;
case "lzo1x999.native":
codecCode = string.Format("LZO1X999.native@{0}", architecture);
compressor = new TimedMethod(codecCode, NativeLZO1X999Compressor);
decompressor = new TimedMethod(codecCode, NativeLZODecompressor);
break;
#endif
case "lz4sharp.unsafe":
codecCode = string.Format("LZ4Sharp.unsafe@{0}", architecture);
if (IntPtr.Size == 4)
{
var lz4sharp_compressor32 = new LZ4Sharp.LZ4Compressor32();
var lz4sharp_decompressor32 = new LZ4Sharp.LZ4Decompressor32();
compressor = new TimedMethod(codecCode, (b, l) => lz4sharp_compressor32.Compress(b));
decompressor = new TimedMethod(codecCode, (b, l) =>
{
var output = new byte[l];
lz4sharp_decompressor32.DecompressKnownSize(b, output);
return output;
});
}
else
{
var lz4sharp_compressor64 = new LZ4Sharp.LZ4Compressor64();
var lz4sharp_decompressor64 = new LZ4Sharp.LZ4Decompressor64();
compressor = new TimedMethod(codecCode, (b, l) => lz4sharp_compressor64.Compress(b));
decompressor = new TimedMethod(codecCode, (b, l) =>
{
var output = new byte[l];
lz4sharp_decompressor64.DecompressKnownSize(b, output);
return output;
});
}
break;
case "quicklz.native":
codecCode = string.Format("QuickLZ.native@{0}", architecture);
compressor = new TimedMethod(codecCode, NativeQuickLZCompressor);
decompressor = new TimedMethod(codecCode, NativeQuickLZDecompressor);
break;
case "snappy.native":
codecCode = string.Format("Snappy.native@{0}", architecture);
compressor = new TimedMethod(codecCode, NativeSnappyCompressor);
decompressor = new TimedMethod(codecCode, NativeSnappyDecompressor);
break;
// safe compressors
case "lz4.safe":
codecCode = string.Format("LZ4.safe@{0}", architecture);
if (IntPtr.Size == 4)
{
compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l));
}
else
{
compressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Encode32(b, 0, l));
decompressor = new TimedMethod(codecCode, (b, l) => LZ4s.LZ4Codec.Decode64(b, 0, b.Length, l));
}
break;
case "lzf.safe":
codecCode = string.Format("LZF.safe@{0}", architecture);
compressor = new TimedMethod(codecCode, SafeLZFCompressor);
decompressor = new TimedMethod(codecCode, SafeLZFDecompressor);
break;
case "quicklz1.safe":
codecCode = string.Format("QuickLZ1.safe@{0}", architecture);
compressor = new TimedMethod(codecCode, SafeQuickLZ1Compressor);
decompressor = new TimedMethod(codecCode, SafeQuickLZDecompressor);
break;
case "quicklz3.safe":
codecCode = string.Format("QuickLZ3.safe@{0}", architecture);
compressor = new TimedMethod(codecCode, SafeQuickLZ3Compressor);
decompressor = new TimedMethod(codecCode, SafeQuickLZDecompressor);
break;
case "deflate.safe":
codecCode = string.Format("Deflate.safe@{0}", architecture);
compressor = new TimedMethod(codecCode, DeflateCompressor);
decompressor = new TimedMethod(codecCode, DeflateDecompressor);
break;
default:
throw new ArgumentException(string.Format("Unknown codec: {0}", codec));
}
return codecCode;
}
private void TestConformance(TimedMethod[] compressors, TimedMethod[] decompressors)
{
var provider = new BlockDataProvider(Utilities.TEST_DATA_FOLDER);
var r = new Random(0);
Console.WriteLine("Architecture: {0}bit", IntPtr.Size * 8);
var total = 0;
const long limit = 1L * 1024 * 1024 * 1024;
var last_pct = 0;
while (total < limit)
{
var length = Utilities.RandomLength(r, MAXIMUM_LENGTH);
var block = provider.GetBytes(length);
TestData(block, compressors, decompressors);
total += block.Length;
var pct = (int)((double)total * 100 / limit);
if (pct > last_pct)
{
Console.WriteLine("{0}%...", pct);
last_pct = pct;
}
}
/*
The performance results from this test are completely unreliable
Too much garbage collection and caching.
So, no need to mislead anyone.
Console.WriteLine("Compression:");
foreach (var compressor in compressors)
{
Console.WriteLine(" {0}: {1:0.00}MB/s", compressor.Name, compressor.Speed);
}
Console.WriteLine("Decompression:");
foreach (var decompressor in decompressors)
{
Console.WriteLine(" {0}: {1:0.00}MB/s", decompressor.Name, decompressor.Speed);
}
*/
}