/// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static unsafe byte[] Pickle( ReadOnlySpan <byte> source, LZ4Level level = LZ4Level.L00_FAST) { var sourceLength = source.Length; if (sourceLength == 0) { return(Mem.Empty); } if (sourceLength <= MAX_STACKALLOC) { var buffer = stackalloc byte[MAX_STACKALLOC]; var target = new Span <byte>(buffer, MAX_STACKALLOC); return(PickleWithBuffer(source, level, target)); } else { var buffer = Mem.Alloc(sourceLength); try { var target = new Span <byte>(buffer, sourceLength); return(PickleWithBuffer(source, level, target)); } finally { Mem.Free(buffer); } } }
public unsafe void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST) { var original = new byte[length]; Lorem.Fill(original, 0, length); var pickled = LZ4Pickler.Pickle(original, level); var unpickled = LZ4Pickler.Unpickle(pickled); Tools.SameBytes(original, unpickled); // reiterating same test, but with different overloads fixed(byte *p = original) pickled = LZ4Pickler.Pickle(p, original.Length, level); fixed(byte *p = pickled) unpickled = LZ4Pickler.Unpickle(p, pickled.Length); Tools.SameBytes(original, unpickled); // reiterating same test, but with offset var copy = new byte[pickled.Length + 37]; Array.Copy(pickled, 0, copy, 37, pickled.Length); unpickled = LZ4Pickler.Unpickle(copy, 37, pickled.Length); Tools.SameBytes(original, unpickled); unpickled.AsSpan().Fill(0); LZ4Pickler.Unpickle(pickled.AsSpan(), unpickled.AsSpan()); Tools.SameBytes(original, unpickled); }
private static byte[] PickleWithBuffer( ReadOnlySpan <byte> source, LZ4Level level, Span <byte> buffer) { const int version = 0; var sourceLength = source.Length; Debug.Assert(buffer.Length >= sourceLength); var encodedLength = LZ4Codec.Encode(source, buffer, level); if (encodedLength <= 0 || encodedLength >= sourceLength) { var headerSize = GetUncompressedHeaderSize(version, sourceLength); var result = new byte[headerSize + sourceLength]; var target = result.AsSpan(); var offset = EncodeUncompressedHeader(target, version, sourceLength); Debug.Assert(headerSize == offset, "Unexpected header size"); source.CopyTo(target.Slice(offset)); return(result); } else { var headerSize = GetCompressedHeaderSize(version, sourceLength, encodedLength); var result = new byte[headerSize + encodedLength]; var target = result.AsSpan(); var offset = EncodeCompressedHeader( target, version, headerSize, sourceLength, encodedLength); Debug.Assert(headerSize == offset, "Unexpected header size"); buffer.Slice(0, encodedLength).CopyTo(target.Slice(offset)); return(result); } }
/// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="sourceLength">Length of input data.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static unsafe byte[] Pickle( byte *source, int sourceLength, LZ4Level level = LZ4Level.L00_FAST) { if (sourceLength <= 0) { return(Array.Empty <byte>()); } var targetLength = sourceLength - 1; var target = (byte *)Mem.Alloc(sourceLength); try { var encodedLength = LZ4Codec.Encode( source, sourceLength, target, targetLength, level); return(encodedLength <= 0 ? PickleV0(source, sourceLength, sourceLength) : PickleV0(target, encodedLength, sourceLength)); } finally { Mem.Free(target); } }
internal static byte[] Encode(byte[] source, LZ4Level level = LZ4Level.L00_FAST) { unsafe { int sourceLength = source.Length; source.Validate(0, sourceLength); fixed (byte* sourceTemp = source) { if (sourceLength <= 0) return Mem.Empty; int targetLength1 = sourceLength - 1; byte* target = (byte*) Mem.Alloc(sourceLength); try { int targetLength2 = LZ4Codec.Encode( sourceTemp, sourceLength, target, targetLength1, level); return targetLength2 <= 0 ? PickleV0(sourceTemp, sourceLength, sourceLength) : PickleV0(target, targetLength2, sourceLength); } finally { Mem.Free((void*) target); } } } }
/// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="sourceOffset">Input buffer offset.</param> /// <param name="sourceLength">Input buffer length.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static unsafe byte[] Pickle( byte[] source, int sourceOffset, int sourceLength, LZ4Level level = LZ4Level.L00_FAST) { source.Validate(sourceOffset, sourceLength); fixed(byte *sourceP = source) return(Pickle(sourceP + sourceOffset, sourceLength, level)); }
public void PickleEntropy(int seed, int length, LZ4Level level = LZ4Level.L00_FAST) { var original = new byte[length]; new Random(seed).NextBytes(original); var pickled = LZ4Pickler.Pickle(original, level); var unpickled = LZ4Pickler.Unpickle(pickled); Tools.SameBytes(original, unpickled); }
public void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST) { var original = new byte[length]; Lorem.Fill(original, 0, length); var pickled = LZ4Pickler.Pickle(original, level); var unpickled = LZ4Pickler.Unpickle(pickled); Tools.SameBytes(original, unpickled); }
public static byte[] Lz4Compress(this byte[] bytes, LZ4Level level = LZ4Level.L09_HC) { var source = bytes.AsSpan(); var target = new byte[LZ4Codec.MaximumOutputSize(source.Length) + 4].AsSpan(); var size = BitConverter.GetBytes(source.Length).AsSpan(); size.CopyTo(target); var compressedBytesSize = LZ4Codec.Encode(source, target.Slice(4), level); return(target.Slice(0, compressedBytesSize + 4).ToArray()); }
/// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static unsafe byte[] Pickle( ReadOnlySpan <byte> source, LZ4Level level = LZ4Level.L00_FAST) { var sourceLength = source.Length; if (sourceLength <= 0) { return(Array.Empty <byte>()); fixed(byte *sourceP = &MemoryMarshal.GetReference(source)) return(Pickle(sourceP, sourceLength, level)); }
/// <summary>Created compression stream on top of inner stream.</summary> /// <param name="stream">Inner stream.</param> /// <param name="level">Compression level.</param> /// <param name="extraMemory">Extra memory used for compression.</param> /// <param name="leaveOpen">Leave inner stream open after disposing.</param> /// <returns>Compression stream.</returns> public static LZ4EncoderStream Encode( Stream stream, LZ4Level level, int extraMemory = 0, bool leaveOpen = false) { var settings = new LZ4EncoderSettings { ChainBlocks = true, ExtraMemory = extraMemory, BlockSize = Mem.K64, CompressionLevel = level }; return(Encode(stream, settings, leaveOpen)); }
/// <summary> /// Serializing an object to binary bois format, then compresses it using LZ4 pickle self-contained format. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="obj"></param> /// <param name="output"></param> /// <param name="lz4Level">Compression level</param> public void Pickle <T>(T obj, Stream output, LZ4Level lz4Level) { using (var mem = new MemoryStream()) { _serializer.Serialize(obj, mem); var serializedBuff = mem.GetBuffer(); var length = (int)mem.Length; var compressedBuff = LZ4Pickler.Pickle(serializedBuff, 0, length, lz4Level); output.Write(compressedBuff, 0, compressedBuff.Length); } }
private static void Roundtrip(string temp, string filename, LZ4Level level) { Console.WriteLine($"Architecture: {IntPtr.Size * 8}bit"); Console.WriteLine($"Roundtrip: {filename} @ {level}..."); Console.WriteLine($"Storage: {temp}"); var originalName = filename; var encodedName = Path.Combine(temp, "encoded.lz4"); var decodedName = Path.Combine(temp, "decoded.lz4"); using (var sourceFile = File.OpenRead(originalName)) using (var targetFile = LZ4Stream.Encode( File.Create(encodedName), level, Mem.M1)) { Console.WriteLine("Compression..."); var stopwatch = Stopwatch.StartNew(); sourceFile.CopyTo(targetFile); stopwatch.Stop(); Console.WriteLine($"Time: {stopwatch.Elapsed.TotalMilliseconds:0.00}ms"); } using (var sourceFile = LZ4Stream.Decode( File.OpenRead(encodedName), Mem.M1)) using (var targetFile = File.Create(decodedName)) { Console.WriteLine("Decompression..."); var stopwatch = Stopwatch.StartNew(); sourceFile.CopyTo(targetFile); stopwatch.Stop(); Console.WriteLine($"Time: {stopwatch.Elapsed.TotalMilliseconds:0.00}ms"); } using (var sourceFile = File.OpenRead(originalName)) using (var targetFile = File.OpenRead(decodedName)) { Console.WriteLine("Verification..."); if (sourceFile.Length != targetFile.Length) { throw new InvalidDataException("Files have different length"); } var sourceChecksum = Checksum(sourceFile); var targetChecksum = Checksum(targetFile); if (sourceChecksum != targetChecksum) { throw new InvalidDataException("Files have different hash"); } } }
/// <summary>Creates new instance of <see cref="LZ4HighChainEncoder"/></summary> /// <param name="level">Compression level.</param> /// <param name="blockSize">Block size.</param> /// <param name="extraBlocks">Number of extra blocks.</param> public LZ4HighChainEncoder(LZ4Level level, int blockSize, int extraBlocks = 0) : base(true, blockSize, extraBlocks) { if (level < LZ4Level.L03_HC) { level = LZ4Level.L03_HC; } if (level > LZ4Level.L12_MAX) { level = LZ4Level.L12_MAX; } _context = (LZ4Context *)Mem.AllocZero(sizeof(LZ4Context)); LZ4_64_HC.LZ4_resetStreamHC(_context, (int)level); }
/// <summary>Creates new instance of <see cref="LZ4HighChainEncoder"/></summary> /// <param name="level">Compression level.</param> /// <param name="blockSize">Block size.</param> /// <param name="extraBlocks">Number of extra blocks.</param> public LZ4HighChainEncoder(LZ4Level level, int blockSize, int extraBlocks = 0) : base(true, blockSize, extraBlocks) { if (level < LZ4Level.L03_HC) { level = LZ4Level.L03_HC; } if (level > LZ4Level.L12_MAX) { level = LZ4Level.L12_MAX; } _context = LL.LZ4_createStreamHC(); LL.LZ4_resetStreamHC_fast(_context, (int)level); }
/// <summary>Compresses data from one buffer into another.</summary> /// <param name="source">Input buffer.</param> /// <param name="sourceOffset">Input buffer offset.</param> /// <param name="sourceLength">Input buffer length.</param> /// <param name="target">Output buffer.</param> /// <param name="targetOffset">Output buffer offset.</param> /// <param name="targetLength">Output buffer length.</param> /// <param name="level">Compression level.</param> /// <returns>Number of bytes written, or negative value if output buffer is too small.</returns> public static unsafe int Encode( byte[] source, int sourceOffset, int sourceLength, byte[] target, int targetOffset, int targetLength, LZ4Level level = LZ4Level.L00_FAST) { source.Validate(sourceOffset, sourceLength); target.Validate(targetOffset, targetLength); fixed(byte *sourceP = source) fixed(byte *targetP = target) return(Encode( sourceP + sourceOffset, sourceLength, targetP + targetOffset, targetLength, level)); }
/// <summary>Compresses data from one buffer into another.</summary> /// <param name="source">Input buffer.</param> /// <param name="target">Output buffer.</param> /// <param name="level">Compression level.</param> /// <returns>Number of bytes written, or negative value if output buffer is too small.</returns> public static unsafe int Encode( ReadOnlySpan <byte> source, Span <byte> target, LZ4Level level = LZ4Level.L00_FAST) { var sourceLength = source.Length; if (sourceLength <= 0) { return(0); } var targetLength = target.Length; fixed(byte *sourceP = &MemoryMarshal.GetReference(source)) fixed(byte *targetP = &MemoryMarshal.GetReference(target)) return(Encode(sourceP, sourceLength, targetP, targetLength, level)); }
public void PickleEntropyWithBufferWriter( int seed, int length, LZ4Level level = LZ4Level.L00_FAST) { var original = new byte[length]; new Random(seed).NextBytes(original); var pickledWriter = BufferWriter.New(); var unpickledWriter = BufferWriter.New(); LZ4Pickler.Pickle(original, pickledWriter, level); var pickled = pickledWriter.WrittenSpan; LZ4Pickler.Unpickle(pickled, unpickledWriter); var unpickled = unpickledWriter.WrittenSpan; Tools.SameBytes(original, unpickled); }
public static byte[] Encode( byte[] source, int sourceIndex, int sourceLength, LZ4Level level) { var bufferLength = LZ4Codec.MaximumOutputSize(sourceLength); var buffer = new byte[bufferLength]; var targetLength = LZ4Codec.Encode( source, sourceIndex, sourceLength, buffer, 0, bufferLength, level); if (targetLength == bufferLength) { return(buffer); } var target = new byte[targetLength]; Buffer.BlockCopy(buffer, 0, target, 0, targetLength); return(target); }
/// <summary>Compresses data from one buffer into another.</summary> /// <param name="source">Input buffer.</param> /// <param name="sourceLength">Length of input buffer.</param> /// <param name="target">Output buffer.</param> /// <param name="targetLength">Output buffer length.</param> /// <param name="level">Compression level.</param> /// <returns>Number of bytes written, or negative value if output buffer is too small.</returns> public static unsafe int Encode( byte *source, int sourceLength, byte *target, int targetLength, LZ4Level level = LZ4Level.L00_FAST) { if (sourceLength <= 0) { return(0); } var encoded = level == LZ4Level.L00_FAST ? LZ4_64.LZ4_compress_default(source, target, sourceLength, targetLength) : LZ4_64_HC.LZ4_compress_HC( source, target, sourceLength, targetLength, (int)level); return(encoded <= 0 ? -1 : encoded); }
/// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="writer">Where the compressed data is written.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static void Pickle( ReadOnlySpan <byte> source, IBufferWriter <byte> writer, LZ4Level level = LZ4Level.L00_FAST) { if (writer is null) { throw new ArgumentNullException(nameof(writer)); } var sourceLength = source.Length; if (sourceLength == 0) { return; } // this might be an argument at some point const int version = 0; // number of bytes is not decided on diff but rather of full length // although, diff would never be greater than full length var headerSize = GetPessimisticHeaderSize(version, sourceLength); var target = writer.GetSpan(headerSize + sourceLength); var encodedLength = LZ4Codec.Encode( source, target.Slice(headerSize, sourceLength), level); if (encodedLength <= 0 || encodedLength >= sourceLength) { var offset = EncodeUncompressedHeader(target, version, sourceLength); source.CopyTo(target.Slice(offset)); writer.Advance(offset + sourceLength); } else { var offset = EncodeCompressedHeader( target, version, headerSize, sourceLength, encodedLength); Debug.Assert(headerSize == offset, "Unexpected header size"); writer.Advance(offset + encodedLength); } }
public static void CREATE_DVPL(LZ4Level COMPRESSION_TYPE, string From_File, string ToFile, bool IsFromFileDelete) { byte[] ORIGINAL_DATA = File.ReadAllBytes(From_File); int ORIGINAL_SIZE = ORIGINAL_DATA.Length; byte[] LZ4_CONTENT = new byte[LZ4Codec.MaximumOutputSize(ORIGINAL_SIZE)]; int LZ4_SIZE = LZ4Codec.Encode(ORIGINAL_DATA, LZ4_CONTENT, COMPRESSION_TYPE); if (COMPRESSION_TYPE == LZ4Level.L00_FAST) { Buffer.BlockCopy(LZ4_CONTENT, 2, LZ4_CONTENT, 0, LZ4_CONTENT.Length - 2); LZ4_SIZE -= 2; } Array.Resize(ref LZ4_CONTENT, LZ4_SIZE); byte[] DVPL_CONTENT = FORMAT_WG_DVPL(LZ4_CONTENT, LZ4_SIZE, ORIGINAL_SIZE, COMPRESSION_TYPE); File.WriteAllBytes(ToFile, DVPL_CONTENT); if (IsFromFileDelete) { File.Delete(From_File); } }
public void PickleLoremWithBufferWriter(int length, LZ4Level level = LZ4Level.L00_FAST) { var original = new byte[length]; Lorem.Fill(original, 0, length); var pickledWriter = BufferWriter.New(); var unpickledWriter = BufferWriter.New(); Assert.Throws <ArgumentNullException>( () => LZ4Pickler.Pickle(original, null, level)); LZ4Pickler.Pickle(original, pickledWriter, level); var pickled = pickledWriter.WrittenSpan; Assert.Throws <ArgumentNullException>( () => LZ4Pickler.Unpickle(pickledWriter.WrittenSpan, (IBufferWriter <byte>)null)); LZ4Pickler.Unpickle(pickled, unpickledWriter); var unpickled = unpickledWriter.WrittenSpan; Tools.SameBytes(original, unpickled); }
public static byte[] Encode( byte[] source, int sourceIndex, int sourceLength, LZ4Level level) { var bufferLength = LZ4Codec.MaximumOutputSize(sourceLength); var buffer = new byte[bufferLength]; var targetLength = LZ4Codec.Encode( source, sourceIndex, sourceLength, buffer, 0, bufferLength, level); if (targetLength <= 0) { throw new ArgumentException("Encoding failed. No bytes returned."); } if (targetLength == bufferLength) { return(buffer); } var target = new byte[targetLength]; Buffer.BlockCopy(buffer, 0, target, 0, targetLength); return(target); }
public LZ4CodecProvider(LZ4Level?level = null) { _level = level ?? LZ4Level.L00_FAST; }
/// <summary> /// Add BrotliNETCompressor to services with specified name /// </summary> /// <param name="services">services</param> /// <param name="name">Name</param> /// <param name="level">LZ4Level</param> /// <returns>IServiceCollection</returns> public static IServiceCollection AddLZ4Compressor(this IServiceCollection services, string name, LZ4Level level = LZ4Level.L12_MAX) { services.TryAddSingleton <ICompressorProvider, DefaultCompressorProvider>(); services.AddSingleton <ICompressor, LZ4Compressor>(x => { return(new LZ4Compressor(name, level)); }); return(services); }
/// <summary> /// Add BrotliNETCompressor to services /// </summary> /// <param name="services">services</param> /// <param name="level">LZ4Level</param> /// <returns>IServiceCollection</returns> public static IServiceCollection AddLZ4Compressor(this IServiceCollection services, LZ4Level level = LZ4Level.L12_MAX) { return(services.AddLZ4Compressor(null, level)); }
private const byte CurrentVersion = 0 & VersionMask; // 3 bits /// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static byte[] Pickle(byte[] source, LZ4Level level = LZ4Level.L00_FAST) => Pickle(source, 0, source.Length, level);
/// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="length">Length of input data.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static unsafe byte[] Pickle( byte *source, int length, LZ4Level level = LZ4Level.L00_FAST) => Pickle(new Span <byte>(source, length), level);
/// <summary>Compresses input buffer into self-contained package.</summary> /// <param name="source">Input buffer.</param> /// <param name="sourceIndex">Input buffer offset.</param> /// <param name="sourceLength">Input buffer length.</param> /// <param name="level">Compression level.</param> /// <returns>Output buffer.</returns> public static byte[] Pickle( byte[] source, int sourceIndex, int sourceLength, LZ4Level level = LZ4Level.L00_FAST) => Pickle(source.AsSpan(sourceIndex, sourceLength), level);