public int Compress(Stream input, Stream output)
{
var length = (int) input.Length;
var varInt = new VarInt32(length).GetEncodedValue();
output.Write(varInt, 0, varInt.Length);
int bytesWritten = varInt.Length;
int bytesToRead = Math.Min(length, CompressorConstants.BlockSize);
var fragment = new byte[bytesToRead];
int maxOutput = MaxCompressedOutput(bytesToRead);
var block = new byte[maxOutput];
while (length > 0)
{
var fragmentSize = input.Read(fragment, 0, bytesToRead);
var hashTable = new HashTable(fragmentSize);
int blockSize = CompressFragment(fragment, fragmentSize, hashTable, block);
output.Write(block, 0, blockSize);
bytesWritten += blockSize;
length -= bytesToRead;
}
return bytesWritten;
}
public static int Compress(Stream input, Stream output)
{
var length = (int)(input.Length - input.Position);
var varInt = new VarInt32(length).GetEncodedValue();
output.Write(varInt, 0, varInt.Length);
int bytesWritten = varInt.Length;
int bytesToRead = Math.Min(length, CompressorConstants.BlockSize);
var fragment = MemoryPool.Instance.Take(bytesToRead);
int maxOutput = MaxCompressedOutput(bytesToRead);
var block = MemoryPool.Instance.Take(maxOutput);
while (length > 0)
{
var fragmentSize = input.Read(fragment, 0, bytesToRead);
var hashTable = new HashTable(fragmentSize);
int blockSize = CompressFragment(fragment, fragmentSize, hashTable, block);
output.Write(block, 0, blockSize);
bytesWritten += blockSize;
length -= bytesToRead;
}
MemoryPool.Instance.Return(fragment);
MemoryPool.Instance.Return(block);
return bytesWritten;
}
private int CompressFragment(byte[] source, int length, HashTable hashTable, byte[] scratchOutput)
{
const int inputMarginBytes = 15;
int shift = 32 - Log2Floor(hashTable.Size);
var op = new Pointer(scratchOutput);
var ip = new Pointer(source);
var nextEmit = new Pointer(source);
Pointer baseIp = new Pointer(ip);
Func<Pointer, int, uint> hashPtr = (value, offset) => (value.ToUInt32(offset) * 0x1e35a7bd) >> shift;
if (length >= inputMarginBytes)
{
var ipLimit = length - inputMarginBytes;
ip = ip + 1;
var nextHash = hashPtr(ip, 0);
while (true)
{
uint skip = 32;
var nextIp = new Pointer(ip);
Pointer candidate;
do
{
ip = nextIp;
uint hash = nextHash;
Assert(hash == hashPtr(ip, 0));
uint bytesBetweenHashLookups = skip++ >> 5;
nextIp = ip + bytesBetweenHashLookups;
if ( nextIp > ipLimit )
{
goto emit_remainder;
}
nextHash = hashPtr(nextIp, 0);
candidate = baseIp + hashTable[hash];
Assert(candidate >= baseIp);
Assert(candidate < ip);
hashTable[hash] = ip - baseIp;
} while (ip.ToUInt32() != candidate.ToUInt32());
Assert(nextEmit + 16 <= length);
op = EmitLiteral(op, nextEmit, ip - nextEmit, allowFastPath: true);
Pointer inputBytes;
uint candidateBytes;
do
{
Pointer b = ip;
int matched = 4 + FindMatchLength(candidate + 4, ip + 4, length);
ip += matched;
var offset = b - candidate;
op = EmitCopy(op, offset, matched);
Pointer insertTail = ip - 1;
nextEmit = ip;
if ( ip >= ipLimit )
{
goto emit_remainder;
}
inputBytes = insertTail;
var prevHash = hashPtr(inputBytes, 0);
hashTable[prevHash] = ip - baseIp - 1;
var curHash = hashPtr(inputBytes, 1);
candidate = baseIp + hashTable[curHash];
candidateBytes = candidate.ToUInt32();
hashTable[curHash] = ip - baseIp;
} while (inputBytes.ToUInt32(1) == candidateBytes);
nextHash = hashPtr(inputBytes, 2);
ip = ip + 1;
}
}
emit_remainder:
if (nextEmit < length)
{
op = EmitLiteral(op, nextEmit, length - nextEmit, false);
}
return op;
}
public void GetHashTable()
{
var hashtable = new HashTable(0);
Assert.That(hashtable.Size, Is.EqualTo(256));
hashtable = new HashTable(CompressorConstants.MaxHashTableSize + 1);
Assert.That(hashtable.Size, Is.EqualTo(CompressorConstants.MaxHashTableSize));
}
private static int CompressFragment(byte[] source, int length, HashTable hashTable, byte[] scratchOutput)
{
const int inputMarginBytes = 15;
int shift = 32 - Log2Floor(hashTable.Size);
var op = new Pointer(scratchOutput);
var ip = new Pointer(source);
var nextEmit = new Pointer(source);
var baseIp = new Pointer(ip);
Func <Pointer, int, uint> hashPtr = (value, offset) => (value.ToUInt32(offset) * 0x1e35a7bd) >> shift;
if (length >= inputMarginBytes)
{
var ipLimit = length - inputMarginBytes;
ip = ip + 1;
var nextHash = hashPtr(ip, 0);
while (true)
{
uint skip = 32;
var nextIp = new Pointer(ip);
Pointer candidate;
do
{
ip = nextIp;
uint hash = nextHash;
Debug.Assert(hash == hashPtr(ip, 0));
uint bytesBetweenHashLookups = skip++ >> 5;
nextIp = ip + bytesBetweenHashLookups;
if (nextIp > ipLimit)
{
goto emit_remainder;
}
nextHash = hashPtr(nextIp, 0);
candidate = baseIp + hashTable[hash];
Debug.Assert(candidate >= baseIp);
Debug.Assert(candidate < ip);
hashTable[hash] = ip - baseIp;
} while (ip.ToUInt32() != candidate.ToUInt32());
Debug.Assert(nextEmit + 16 <= length);
op = EmitLiteral(op, nextEmit, ip - nextEmit, allowFastPath: true);
Pointer inputBytes;
uint candidateBytes;
do
{
Pointer b = ip;
int matched = 4 + FindMatchLength(candidate + 4, ip + 4, length);
ip += matched;
var offset = b - candidate;
op = EmitCopy(op, offset, matched);
Pointer insertTail = ip - 1;
nextEmit = ip;
if (ip >= ipLimit)
{
goto emit_remainder;
}
inputBytes = insertTail;
var prevHash = hashPtr(inputBytes, 0);
hashTable[prevHash] = ip - baseIp - 1;
var curHash = hashPtr(inputBytes, 1);
candidate = baseIp + hashTable[curHash];
candidateBytes = candidate.ToUInt32();
hashTable[curHash] = ip - baseIp;
} while (inputBytes.ToUInt32(1) == candidateBytes);
nextHash = hashPtr(inputBytes, 2);
ip = ip + 1;
}
}
emit_remainder:
if (nextEmit < length)
{
op = EmitLiteral(op, nextEmit, length - nextEmit, false);
}
return(op);
}
