public void CompressionImprovesWithDictionary()
{
var trainingData = new byte[100][];
for (int i = 0; i < trainingData.Length; i++)
{
trainingData[i] = DataGenerator.GetSmallBuffer(DataFill.Random);
}
var dict = DictBuilder.TrainFromBuffer(trainingData);
var compressionOptions = new CompressionOptions(dict);
var testStream = DataGenerator.GetSmallStream(DataFill.Random);
var normalResultStream = new MemoryStream();
using (var compressionStream = new CompressorStream(normalResultStream))
testStream.CopyTo(compressionStream);
var dictResultStream = new MemoryStream();
using (var compressionStream = new CompressorStream(dictResultStream, compressionOptions))
testStream.CopyTo(compressionStream);
Assert.Greater(normalResultStream.Length, dictResultStream.Length);
}
private static byte[] TrainDict()
{
var trainingData = new byte[100][];
for (int i = 0; i < trainingData.Length; i++)
{
trainingData[i] = DataGenerator.GetSmallBuffer(DataFill.Sequential);
}
return(DictBuilder.TrainFromBuffer(trainingData));
}
public void TrainDictionaryParallel()
{
var buffer = Enumerable.Range(0, 100000).Select(i => unchecked ((byte)(i * i))).ToArray();
var samples = Enumerable.Range(0, 100)
.Select(i => buffer.Skip(i).Take(200 - i).ToArray())
.ToArray();
var dict = DictBuilder.TrainFromBuffer(samples);
Assert.Greater(dict.Length, 0);
Assert.LessOrEqual(dict.Length, DictBuilder.DefaultDictCapacity);
Enumerable.Range(0, 100000)
.AsParallel().WithDegreeOfParallelism(Environment.ProcessorCount * 4)
.ForAll(_ => Assert.IsTrue(dict.SequenceEqual(DictBuilder.TrainFromBuffer(samples))));
}
private static byte[] BuildDictionary() => DictBuilder.TrainFromBuffer(Enumerable.Range(0, 8).Select(_ => GenerateSample()).ToArray(), 1024);
