public void CompressionImprovesWithDictionary() { var trainingData = new byte[100][]; for (int i = 0; i < trainingData.Length; i++) { trainingData[i] = DataGenerator.GetSmallBuffer(DataFill.Random); } var dict = DictBuilder.TrainFromBuffer(trainingData); var compressionOptions = new CompressionOptions(dict); var testStream = DataGenerator.GetSmallStream(DataFill.Random); var normalResultStream = new MemoryStream(); using (var compressionStream = new CompressorStream(normalResultStream)) testStream.CopyTo(compressionStream); var dictResultStream = new MemoryStream(); using (var compressionStream = new CompressorStream(dictResultStream, compressionOptions)) testStream.CopyTo(compressionStream); Assert.Greater(normalResultStream.Length, dictResultStream.Length); }
private static byte[] TrainDict() { var trainingData = new byte[100][]; for (int i = 0; i < trainingData.Length; i++) { trainingData[i] = DataGenerator.GetSmallBuffer(DataFill.Sequential); } return(DictBuilder.TrainFromBuffer(trainingData)); }
public void TrainDictionaryParallel() { var buffer = Enumerable.Range(0, 100000).Select(i => unchecked ((byte)(i * i))).ToArray(); var samples = Enumerable.Range(0, 100) .Select(i => buffer.Skip(i).Take(200 - i).ToArray()) .ToArray(); var dict = DictBuilder.TrainFromBuffer(samples); Assert.Greater(dict.Length, 0); Assert.LessOrEqual(dict.Length, DictBuilder.DefaultDictCapacity); Enumerable.Range(0, 100000) .AsParallel().WithDegreeOfParallelism(Environment.ProcessorCount * 4) .ForAll(_ => Assert.IsTrue(dict.SequenceEqual(DictBuilder.TrainFromBuffer(samples)))); }
private static byte[] BuildDictionary() => DictBuilder.TrainFromBuffer(Enumerable.Range(0, 8).Select(_ => GenerateSample()).ToArray(), 1024);