public static int Compress(Stream input, Stream output) { var length = (int)(input.Length - input.Position); var varInt = new VarInt32(length).GetEncodedValue(); output.Write(varInt, 0, varInt.Length); int bytesWritten = varInt.Length; int bytesToRead = Math.Min(length, CompressorConstants.BlockSize); var fragment = MemoryPool.Instance.Take(bytesToRead); int maxOutput = MaxCompressedOutput(bytesToRead); var block = MemoryPool.Instance.Take(maxOutput); while(length > 0) { var fragmentSize = input.Read(fragment, 0, bytesToRead); var hashTable = new HashTable(fragmentSize); int blockSize = CompressFragment(fragment, fragmentSize, hashTable, block); output.Write(block, 0, blockSize); bytesWritten += blockSize; length -= bytesToRead; } MemoryPool.Instance.Return(fragment); MemoryPool.Instance.Return(block); return bytesWritten; }
private static int CompressFragment(byte[] source, int length, HashTable hashTable, byte[] scratchOutput) { const int inputMarginBytes = 15; int shift = 32 - Log2Floor(hashTable.Size); var op = new Pointer(scratchOutput); var ip = new Pointer(source); var nextEmit = new Pointer(source); var baseIp = new Pointer(ip); Func<Pointer, int, uint> hashPtr = (value, offset) => (value.ToUInt32(offset)*0x1e35a7bd) >> shift; if(length >= inputMarginBytes) { var ipLimit = length - inputMarginBytes; ip = ip + 1; var nextHash = hashPtr(ip, 0); while(true) { uint skip = 32; var nextIp = new Pointer(ip); Pointer candidate; do { ip = nextIp; uint hash = nextHash; Debug.Assert(hash == hashPtr(ip, 0)); uint bytesBetweenHashLookups = skip++ >> 5; nextIp = ip + bytesBetweenHashLookups; if(nextIp > ipLimit) goto emit_remainder; nextHash = hashPtr(nextIp, 0); candidate = baseIp + hashTable[hash]; Debug.Assert(candidate >= baseIp); Debug.Assert(candidate < ip); hashTable[hash] = ip - baseIp; } while(ip.ToUInt32() != candidate.ToUInt32()); Debug.Assert(nextEmit + 16 <= length); op = EmitLiteral(op, nextEmit, ip - nextEmit, true); Pointer inputBytes; uint candidateBytes; do { Pointer b = ip; int matched = 4 + FindMatchLength(candidate + 4, ip + 4, length); ip += matched; var offset = b - candidate; op = EmitCopy(op, offset, matched); Pointer insertTail = ip - 1; nextEmit = ip; if(ip >= ipLimit) goto emit_remainder; inputBytes = insertTail; var prevHash = hashPtr(inputBytes, 0); hashTable[prevHash] = ip - baseIp - 1; var curHash = hashPtr(inputBytes, 1); candidate = baseIp + hashTable[curHash]; candidateBytes = candidate.ToUInt32(); hashTable[curHash] = ip - baseIp; } while(inputBytes.ToUInt32(1) == candidateBytes); nextHash = hashPtr(inputBytes, 2); ip = ip + 1; } } emit_remainder: if(nextEmit < length) op = EmitLiteral(op, nextEmit, length - nextEmit, false); return op; }