/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static unsafe byte[] Pickle(
ReadOnlySpan <byte> source, LZ4Level level = LZ4Level.L00_FAST)
{
var sourceLength = source.Length;
if (sourceLength == 0)
{
return(Mem.Empty);
}
if (sourceLength <= MAX_STACKALLOC)
{
var buffer = stackalloc byte[MAX_STACKALLOC];
var target = new Span <byte>(buffer, MAX_STACKALLOC);
return(PickleWithBuffer(source, level, target));
}
else
{
var buffer = Mem.Alloc(sourceLength);
try
{
var target = new Span <byte>(buffer, sourceLength);
return(PickleWithBuffer(source, level, target));
}
finally
{
Mem.Free(buffer);
}
}
}
public unsafe void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
Lorem.Fill(original, 0, length);
var pickled = LZ4Pickler.Pickle(original, level);
var unpickled = LZ4Pickler.Unpickle(pickled);
Tools.SameBytes(original, unpickled);
// reiterating same test, but with different overloads
fixed(byte *p = original)
pickled = LZ4Pickler.Pickle(p, original.Length, level);
fixed(byte *p = pickled)
unpickled = LZ4Pickler.Unpickle(p, pickled.Length);
Tools.SameBytes(original, unpickled);
// reiterating same test, but with offset
var copy = new byte[pickled.Length + 37];
Array.Copy(pickled, 0, copy, 37, pickled.Length);
unpickled = LZ4Pickler.Unpickle(copy, 37, pickled.Length);
Tools.SameBytes(original, unpickled);
unpickled.AsSpan().Fill(0);
LZ4Pickler.Unpickle(pickled.AsSpan(), unpickled.AsSpan());
Tools.SameBytes(original, unpickled);
}
private static byte[] PickleWithBuffer(
ReadOnlySpan <byte> source, LZ4Level level, Span <byte> buffer)
{
const int version = 0;
var sourceLength = source.Length;
Debug.Assert(buffer.Length >= sourceLength);
var encodedLength = LZ4Codec.Encode(source, buffer, level);
if (encodedLength <= 0 || encodedLength >= sourceLength)
{
var headerSize = GetUncompressedHeaderSize(version, sourceLength);
var result = new byte[headerSize + sourceLength];
var target = result.AsSpan();
var offset = EncodeUncompressedHeader(target, version, sourceLength);
Debug.Assert(headerSize == offset, "Unexpected header size");
source.CopyTo(target.Slice(offset));
return(result);
}
else
{
var headerSize = GetCompressedHeaderSize(version, sourceLength, encodedLength);
var result = new byte[headerSize + encodedLength];
var target = result.AsSpan();
var offset = EncodeCompressedHeader(
target, version, headerSize, sourceLength, encodedLength);
Debug.Assert(headerSize == offset, "Unexpected header size");
buffer.Slice(0, encodedLength).CopyTo(target.Slice(offset));
return(result);
}
}
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="sourceLength">Length of input data.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static unsafe byte[] Pickle(
byte *source, int sourceLength, LZ4Level level = LZ4Level.L00_FAST)
{
if (sourceLength <= 0)
{
return(Array.Empty <byte>());
}
var targetLength = sourceLength - 1;
var target = (byte *)Mem.Alloc(sourceLength);
try
{
var encodedLength = LZ4Codec.Encode(
source, sourceLength, target, targetLength, level);
return(encodedLength <= 0
? PickleV0(source, sourceLength, sourceLength)
: PickleV0(target, encodedLength, sourceLength));
}
finally
{
Mem.Free(target);
}
}
internal static byte[] Encode(byte[] source, LZ4Level level = LZ4Level.L00_FAST)
{
unsafe
{
int sourceLength = source.Length;
source.Validate(0, sourceLength);
fixed (byte* sourceTemp = source)
{
if (sourceLength <= 0) return Mem.Empty;
int targetLength1 = sourceLength - 1;
byte* target = (byte*) Mem.Alloc(sourceLength);
try
{
int targetLength2 = LZ4Codec.Encode(
sourceTemp, sourceLength,
target, targetLength1,
level);
return targetLength2 <= 0
? PickleV0(sourceTemp, sourceLength, sourceLength)
: PickleV0(target, targetLength2, sourceLength);
}
finally
{
Mem.Free((void*) target);
}
}
}
}
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="sourceOffset">Input buffer offset.</param>
/// <param name="sourceLength">Input buffer length.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static unsafe byte[] Pickle(
byte[] source, int sourceOffset, int sourceLength,
LZ4Level level = LZ4Level.L00_FAST)
{
source.Validate(sourceOffset, sourceLength);
fixed(byte *sourceP = source)
return(Pickle(sourceP + sourceOffset, sourceLength, level));
}
public void PickleEntropy(int seed, int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
new Random(seed).NextBytes(original);
var pickled = LZ4Pickler.Pickle(original, level);
var unpickled = LZ4Pickler.Unpickle(pickled);
Tools.SameBytes(original, unpickled);
}
public void PickleLorem(int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
Lorem.Fill(original, 0, length);
var pickled = LZ4Pickler.Pickle(original, level);
var unpickled = LZ4Pickler.Unpickle(pickled);
Tools.SameBytes(original, unpickled);
}
public static byte[] Lz4Compress(this byte[] bytes, LZ4Level level = LZ4Level.L09_HC)
{
var source = bytes.AsSpan();
var target = new byte[LZ4Codec.MaximumOutputSize(source.Length) + 4].AsSpan();
var size = BitConverter.GetBytes(source.Length).AsSpan();
size.CopyTo(target);
var compressedBytesSize = LZ4Codec.Encode(source, target.Slice(4), level);
return(target.Slice(0, compressedBytesSize + 4).ToArray());
}
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static unsafe byte[] Pickle(
ReadOnlySpan <byte> source, LZ4Level level = LZ4Level.L00_FAST)
{
var sourceLength = source.Length;
if (sourceLength <= 0)
{
return(Array.Empty <byte>());
fixed(byte *sourceP = &MemoryMarshal.GetReference(source))
return(Pickle(sourceP, sourceLength, level));
}
/// <summary>Created compression stream on top of inner stream.</summary>
/// <param name="stream">Inner stream.</param>
/// <param name="level">Compression level.</param>
/// <param name="extraMemory">Extra memory used for compression.</param>
/// <param name="leaveOpen">Leave inner stream open after disposing.</param>
/// <returns>Compression stream.</returns>
public static LZ4EncoderStream Encode(
Stream stream, LZ4Level level, int extraMemory = 0,
bool leaveOpen = false)
{
var settings = new LZ4EncoderSettings {
ChainBlocks = true,
ExtraMemory = extraMemory,
BlockSize = Mem.K64,
CompressionLevel = level
};
return(Encode(stream, settings, leaveOpen));
}
/// <summary>
/// Serializing an object to binary bois format, then compresses it using LZ4 pickle self-contained format.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="obj"></param>
/// <param name="output"></param>
/// <param name="lz4Level">Compression level</param>
public void Pickle <T>(T obj, Stream output, LZ4Level lz4Level)
{
using (var mem = new MemoryStream())
{
_serializer.Serialize(obj, mem);
var serializedBuff = mem.GetBuffer();
var length = (int)mem.Length;
var compressedBuff = LZ4Pickler.Pickle(serializedBuff, 0, length, lz4Level);
output.Write(compressedBuff, 0, compressedBuff.Length);
}
}
private static void Roundtrip(string temp, string filename, LZ4Level level)
{
Console.WriteLine($"Architecture: {IntPtr.Size * 8}bit");
Console.WriteLine($"Roundtrip: {filename} @ {level}...");
Console.WriteLine($"Storage: {temp}");
var originalName = filename;
var encodedName = Path.Combine(temp, "encoded.lz4");
var decodedName = Path.Combine(temp, "decoded.lz4");
using (var sourceFile = File.OpenRead(originalName))
using (var targetFile = LZ4Stream.Encode(
File.Create(encodedName), level, Mem.M1))
{
Console.WriteLine("Compression...");
var stopwatch = Stopwatch.StartNew();
sourceFile.CopyTo(targetFile);
stopwatch.Stop();
Console.WriteLine($"Time: {stopwatch.Elapsed.TotalMilliseconds:0.00}ms");
}
using (var sourceFile = LZ4Stream.Decode(
File.OpenRead(encodedName), Mem.M1))
using (var targetFile = File.Create(decodedName))
{
Console.WriteLine("Decompression...");
var stopwatch = Stopwatch.StartNew();
sourceFile.CopyTo(targetFile);
stopwatch.Stop();
Console.WriteLine($"Time: {stopwatch.Elapsed.TotalMilliseconds:0.00}ms");
}
using (var sourceFile = File.OpenRead(originalName))
using (var targetFile = File.OpenRead(decodedName))
{
Console.WriteLine("Verification...");
if (sourceFile.Length != targetFile.Length)
{
throw new InvalidDataException("Files have different length");
}
var sourceChecksum = Checksum(sourceFile);
var targetChecksum = Checksum(targetFile);
if (sourceChecksum != targetChecksum)
{
throw new InvalidDataException("Files have different hash");
}
}
}
/// <summary>Creates new instance of <see cref="LZ4HighChainEncoder"/></summary>
/// <param name="level">Compression level.</param>
/// <param name="blockSize">Block size.</param>
/// <param name="extraBlocks">Number of extra blocks.</param>
public LZ4HighChainEncoder(LZ4Level level, int blockSize, int extraBlocks = 0) :
base(true, blockSize, extraBlocks)
{
if (level < LZ4Level.L03_HC)
{
level = LZ4Level.L03_HC;
}
if (level > LZ4Level.L12_MAX)
{
level = LZ4Level.L12_MAX;
}
_context = (LZ4Context *)Mem.AllocZero(sizeof(LZ4Context));
LZ4_64_HC.LZ4_resetStreamHC(_context, (int)level);
}
/// <summary>Creates new instance of <see cref="LZ4HighChainEncoder"/></summary>
/// <param name="level">Compression level.</param>
/// <param name="blockSize">Block size.</param>
/// <param name="extraBlocks">Number of extra blocks.</param>
public LZ4HighChainEncoder(LZ4Level level, int blockSize, int extraBlocks = 0) :
base(true, blockSize, extraBlocks)
{
if (level < LZ4Level.L03_HC)
{
level = LZ4Level.L03_HC;
}
if (level > LZ4Level.L12_MAX)
{
level = LZ4Level.L12_MAX;
}
_context = LL.LZ4_createStreamHC();
LL.LZ4_resetStreamHC_fast(_context, (int)level);
}
/// <summary>Compresses data from one buffer into another.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="sourceOffset">Input buffer offset.</param>
/// <param name="sourceLength">Input buffer length.</param>
/// <param name="target">Output buffer.</param>
/// <param name="targetOffset">Output buffer offset.</param>
/// <param name="targetLength">Output buffer length.</param>
/// <param name="level">Compression level.</param>
/// <returns>Number of bytes written, or negative value if output buffer is too small.</returns>
public static unsafe int Encode(
byte[] source, int sourceOffset, int sourceLength,
byte[] target, int targetOffset, int targetLength,
LZ4Level level = LZ4Level.L00_FAST)
{
source.Validate(sourceOffset, sourceLength);
target.Validate(targetOffset, targetLength);
fixed(byte *sourceP = source)
fixed(byte *targetP = target)
return(Encode(
sourceP + sourceOffset, sourceLength,
targetP + targetOffset, targetLength,
level));
}
/// <summary>Compresses data from one buffer into another.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="target">Output buffer.</param>
/// <param name="level">Compression level.</param>
/// <returns>Number of bytes written, or negative value if output buffer is too small.</returns>
public static unsafe int Encode(
ReadOnlySpan <byte> source, Span <byte> target,
LZ4Level level = LZ4Level.L00_FAST)
{
var sourceLength = source.Length;
if (sourceLength <= 0)
{
return(0);
}
var targetLength = target.Length;
fixed(byte *sourceP = &MemoryMarshal.GetReference(source))
fixed(byte *targetP = &MemoryMarshal.GetReference(target))
return(Encode(sourceP, sourceLength, targetP, targetLength, level));
}
public void PickleEntropyWithBufferWriter(
int seed, int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
new Random(seed).NextBytes(original);
var pickledWriter = BufferWriter.New();
var unpickledWriter = BufferWriter.New();
LZ4Pickler.Pickle(original, pickledWriter, level);
var pickled = pickledWriter.WrittenSpan;
LZ4Pickler.Unpickle(pickled, unpickledWriter);
var unpickled = unpickledWriter.WrittenSpan;
Tools.SameBytes(original, unpickled);
}
public static byte[] Encode(
byte[] source, int sourceIndex, int sourceLength, LZ4Level level)
{
var bufferLength = LZ4Codec.MaximumOutputSize(sourceLength);
var buffer = new byte[bufferLength];
var targetLength = LZ4Codec.Encode(
source, sourceIndex, sourceLength, buffer, 0, bufferLength, level);
if (targetLength == bufferLength)
{
return(buffer);
}
var target = new byte[targetLength];
Buffer.BlockCopy(buffer, 0, target, 0, targetLength);
return(target);
}
/// <summary>Compresses data from one buffer into another.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="sourceLength">Length of input buffer.</param>
/// <param name="target">Output buffer.</param>
/// <param name="targetLength">Output buffer length.</param>
/// <param name="level">Compression level.</param>
/// <returns>Number of bytes written, or negative value if output buffer is too small.</returns>
public static unsafe int Encode(
byte *source, int sourceLength,
byte *target, int targetLength,
LZ4Level level = LZ4Level.L00_FAST)
{
if (sourceLength <= 0)
{
return(0);
}
var encoded =
level == LZ4Level.L00_FAST
? LZ4_64.LZ4_compress_default(source, target, sourceLength, targetLength)
: LZ4_64_HC.LZ4_compress_HC(
source, target, sourceLength, targetLength, (int)level);
return(encoded <= 0 ? -1 : encoded);
}
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="writer">Where the compressed data is written.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static void Pickle(
ReadOnlySpan <byte> source, IBufferWriter <byte> writer,
LZ4Level level = LZ4Level.L00_FAST)
{
if (writer is null)
{
throw new ArgumentNullException(nameof(writer));
}
var sourceLength = source.Length;
if (sourceLength == 0)
{
return;
}
// this might be an argument at some point
const int version = 0;
// number of bytes is not decided on diff but rather of full length
// although, diff would never be greater than full length
var headerSize = GetPessimisticHeaderSize(version, sourceLength);
var target = writer.GetSpan(headerSize + sourceLength);
var encodedLength = LZ4Codec.Encode(
source, target.Slice(headerSize, sourceLength), level);
if (encodedLength <= 0 || encodedLength >= sourceLength)
{
var offset = EncodeUncompressedHeader(target, version, sourceLength);
source.CopyTo(target.Slice(offset));
writer.Advance(offset + sourceLength);
}
else
{
var offset = EncodeCompressedHeader(
target, version, headerSize, sourceLength, encodedLength);
Debug.Assert(headerSize == offset, "Unexpected header size");
writer.Advance(offset + encodedLength);
}
}
public static void CREATE_DVPL(LZ4Level COMPRESSION_TYPE, string From_File, string ToFile, bool IsFromFileDelete)
{
byte[] ORIGINAL_DATA = File.ReadAllBytes(From_File);
int ORIGINAL_SIZE = ORIGINAL_DATA.Length;
byte[] LZ4_CONTENT = new byte[LZ4Codec.MaximumOutputSize(ORIGINAL_SIZE)];
int LZ4_SIZE = LZ4Codec.Encode(ORIGINAL_DATA, LZ4_CONTENT, COMPRESSION_TYPE);
if (COMPRESSION_TYPE == LZ4Level.L00_FAST)
{
Buffer.BlockCopy(LZ4_CONTENT, 2, LZ4_CONTENT, 0, LZ4_CONTENT.Length - 2);
LZ4_SIZE -= 2;
}
Array.Resize(ref LZ4_CONTENT, LZ4_SIZE);
byte[] DVPL_CONTENT = FORMAT_WG_DVPL(LZ4_CONTENT, LZ4_SIZE, ORIGINAL_SIZE, COMPRESSION_TYPE);
File.WriteAllBytes(ToFile, DVPL_CONTENT);
if (IsFromFileDelete)
{
File.Delete(From_File);
}
}
public void PickleLoremWithBufferWriter(int length, LZ4Level level = LZ4Level.L00_FAST)
{
var original = new byte[length];
Lorem.Fill(original, 0, length);
var pickledWriter = BufferWriter.New();
var unpickledWriter = BufferWriter.New();
Assert.Throws <ArgumentNullException>(
() => LZ4Pickler.Pickle(original, null, level));
LZ4Pickler.Pickle(original, pickledWriter, level);
var pickled = pickledWriter.WrittenSpan;
Assert.Throws <ArgumentNullException>(
() => LZ4Pickler.Unpickle(pickledWriter.WrittenSpan, (IBufferWriter <byte>)null));
LZ4Pickler.Unpickle(pickled, unpickledWriter);
var unpickled = unpickledWriter.WrittenSpan;
Tools.SameBytes(original, unpickled);
}
public static byte[] Encode(
byte[] source, int sourceIndex, int sourceLength, LZ4Level level)
{
var bufferLength = LZ4Codec.MaximumOutputSize(sourceLength);
var buffer = new byte[bufferLength];
var targetLength = LZ4Codec.Encode(
source, sourceIndex, sourceLength, buffer, 0, bufferLength, level);
if (targetLength <= 0)
{
throw new ArgumentException("Encoding failed. No bytes returned.");
}
if (targetLength == bufferLength)
{
return(buffer);
}
var target = new byte[targetLength];
Buffer.BlockCopy(buffer, 0, target, 0, targetLength);
return(target);
}
public LZ4CodecProvider(LZ4Level?level = null)
{
_level = level ?? LZ4Level.L00_FAST;
}
/// <summary>
/// Add BrotliNETCompressor to services with specified name
/// </summary>
/// <param name="services">services</param>
/// <param name="name">Name</param>
/// <param name="level">LZ4Level</param>
/// <returns>IServiceCollection</returns>
public static IServiceCollection AddLZ4Compressor(this IServiceCollection services, string name, LZ4Level level = LZ4Level.L12_MAX)
{
services.TryAddSingleton <ICompressorProvider, DefaultCompressorProvider>();
services.AddSingleton <ICompressor, LZ4Compressor>(x =>
{
return(new LZ4Compressor(name, level));
});
return(services);
}
/// <summary>
/// Add BrotliNETCompressor to services
/// </summary>
/// <param name="services">services</param>
/// <param name="level">LZ4Level</param>
/// <returns>IServiceCollection</returns>
public static IServiceCollection AddLZ4Compressor(this IServiceCollection services, LZ4Level level = LZ4Level.L12_MAX)
{
return(services.AddLZ4Compressor(null, level));
}
private const byte CurrentVersion = 0 & VersionMask; // 3 bits
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static byte[] Pickle(byte[] source, LZ4Level level = LZ4Level.L00_FAST) =>
Pickle(source, 0, source.Length, level);
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="length">Length of input data.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static unsafe byte[] Pickle(
byte *source, int length, LZ4Level level = LZ4Level.L00_FAST) =>
Pickle(new Span <byte>(source, length), level);
/// <summary>Compresses input buffer into self-contained package.</summary>
/// <param name="source">Input buffer.</param>
/// <param name="sourceIndex">Input buffer offset.</param>
/// <param name="sourceLength">Input buffer length.</param>
/// <param name="level">Compression level.</param>
/// <returns>Output buffer.</returns>
public static byte[] Pickle(
byte[] source, int sourceIndex, int sourceLength,
LZ4Level level = LZ4Level.L00_FAST) =>
Pickle(source.AsSpan(sourceIndex, sourceLength), level);
