[InlineData(0)] // special case : in = Exactly 1GB, out > 1GB
public unsafe void LZ4TestAbove2GB(long divider)
{
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(10)))
{
var options = StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, $"bigLz4-test-{divider}.data"));
using (var env = new StorageEnvironment(options))
{
long gb = 1024 * 1024 * 1024;
long inputSize = 3L * gb;
var guid = Guid.NewGuid();
using (var outputPager = CreateScratchFile($"output-{divider}-{guid}", env, inputSize, out byte *outputBuffer))
using (var inputPager = CreateScratchFile($"input-{divider}-{guid}", env, inputSize, out byte *inputBuffer))
using (var checkedPager = CreateScratchFile($"checked-{divider}-{guid}", env, inputSize, out byte *checkedBuffer))
{
var random = new Random(123);
if (divider != 0)
{
for (long p = 0; p < inputSize / divider; p++)
{
cts.Token.ThrowIfCancellationRequested();
(*(byte *)((long)inputBuffer + p)) = Convert.ToByte(random.Next(0, 255));
}
}
else
{
inputSize = int.MaxValue / 2 - 1; // MAX_INPUT_LENGTH_PER_SEGMENT
for (long p = 0; p < inputSize; p++)
{
cts.Token.ThrowIfCancellationRequested();
(*(byte *)((long)inputBuffer + p)) = Convert.ToByte(random.Next(0, 255));
}
}
Console.WriteLine("Calculating LZ4 MaximumOutputLength...");
var outputBufferSize = LZ4.MaximumOutputLength(inputSize);
Console.WriteLine("...done");
// write some data in known places in inputBuffer
byte testNum = 0;
for (long testPoints = 0; testPoints < inputSize; testPoints += gb)
{
cts.Token.ThrowIfCancellationRequested();
var testPointer = (byte *)((long)inputBuffer + testPoints);
* testPointer = ++testNum;
}
Console.WriteLine("Encoding LZ4 LongBuffer...");
// encode inputBuffer into outputBuffer
var compressedLenTask = Task.Factory.StartNew(() => LZ4.Encode64LongBuffer(inputBuffer, outputBuffer, inputSize, outputBufferSize), cts.Token);
compressedLenTask.Wait(cts.Token);
var compressedLen = compressedLenTask.Result;
Console.WriteLine("...done");
Console.WriteLine("Decoding LZ4 LongBuffers...");
// decode outputBuffer into checkedBuffer
var totalOutputSizeTask = Task.Factory.StartNew(() => LZ4.Decode64LongBuffers(outputBuffer, compressedLen, checkedBuffer, inputSize, true), cts.Token);
totalOutputSizeTask.Wait(cts.Token);
var totalOutputSize = totalOutputSizeTask.Result;
Console.WriteLine("...done");
Assert.Equal(compressedLen, totalOutputSize);
testNum = 0;
for (long testPoints = 0; testPoints < inputSize; testPoints += gb)
{
cts.Token.ThrowIfCancellationRequested();
var testPointer = (byte *)((long)checkedBuffer + testPoints);
Assert.Equal(++testNum, *testPointer);
}
}
}
}
}
[InlineData(0)] // special case : in = Exactly 1GB, out > 1GB
public unsafe void LZ4TestAbove2GB(long devider)
{
var options = StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, $"bigLz4-test-{devider}.data"));
using (var env = new StorageEnvironment(options))
{
long Gb = 1024 * 1024 * 1024;
long inputSize = 3L * Gb;
byte *outputBuffer, inputBuffer, checkedBuffer;
var guid = Guid.NewGuid();
var outputPager = CreateScratchFile($"output-{devider}-{guid}", env, inputSize, out outputBuffer);
var inputPager = CreateScratchFile($"input-{devider}-{guid}", env, inputSize, out inputBuffer);
var checkedPager = CreateScratchFile($"checked-{devider}-{guid}", env, inputSize, out checkedBuffer);
var random = new Random(123);
if (devider != 0)
{
for (long p = 0; p < inputSize / devider; p++)
{
(*(byte *)((long)inputBuffer + p)) = Convert.ToByte(random.Next(0, 255));
}
}
else
{
inputSize = int.MaxValue / 2 - 1; // MAX_INPUT_LENGTH_PER_SEGMENT
for (long p = 0; p < inputSize; p++)
{
(*(byte *)((long)inputBuffer + p)) = Convert.ToByte(random.Next(0, 255));
}
}
var outputBufferSize = LZ4.MaximumOutputLength(inputSize);
// write some data in known places in inputBuffer
long compressedLen = 0;
byte testNum = 0;
for (long testPoints = 0; testPoints < inputSize; testPoints += Gb)
{
var testPointer = (byte *)((long)inputBuffer + testPoints);
* testPointer = ++testNum;
}
// encode inputBuffer into outputBuffer
compressedLen = LZ4.Encode64LongBuffer(inputBuffer, outputBuffer, inputSize, outputBufferSize);
// decode outputBuffer into checkedBuffer
var totalOutputSize = LZ4.Decode64LongBuffers(outputBuffer, compressedLen, checkedBuffer, inputSize, true);
Assert.Equal(compressedLen, totalOutputSize);
testNum = 0;
for (long testPoints = 0; testPoints < inputSize; testPoints += Gb)
{
var testPointer = (byte *)((long)checkedBuffer + testPoints);
Assert.Equal(++testNum, *testPointer);
}
outputPager.Dispose();
inputPager.Dispose();
checkedPager.Dispose();
}
}
