public static int CompressFastExtState(StreamT *state, byte *source, byte *dest, int inputSize, int maxOutputSize, int acceleration) { ResetStream(state); if (acceleration < 1) { acceleration = AccelerationDefault; } LimitedOutputDirective limited = maxOutputSize >= CompressBound(inputSize) ? LimitedOutputDirective.NoLimit : LimitedOutputDirective.LimitedOutput; return(CompressGeneric( state, source, dest, inputSize, limited == LimitedOutputDirective.NoLimit ? 0 : maxOutputSize, limited, TableType(inputSize), DictDirective.NoDict, DictIssueDirective.NoDictIssue, (uint)acceleration)); }
public static int CompressGeneric(StreamT *cctx, byte *source, byte *dest, int inputSize, int maxOutputSize, LimitedOutputDirective outputLimited, TableTypeT tableType, DictDirective dict, DictIssueDirective dictIssue, uint acceleration) { byte *ip = source; byte *ibase; byte *lowLimit; byte *lowRefLimit = ip - cctx->DictSize; byte *dictionary = cctx->Dictionary; byte *dictEnd = dictionary + cctx->DictSize; long dictDelta = dictEnd - source; byte *anchor = source; byte *iend = ip + inputSize; byte *mflimit = iend - MFLimit; byte *matchlimit = iend - LastLiterals; byte *op = dest; byte *olimit = op + maxOutputSize; if (inputSize > MaxInputSize) { return(0); } switch (dict) { case DictDirective.WithPrefix64k: ibase = source - cctx->CurrentOffset; lowLimit = source - cctx->DictSize; break; case DictDirective.UsingExtDict: ibase = source - cctx->CurrentOffset; lowLimit = source; break; default: ibase = source; lowLimit = source; break; } if ((tableType == TableTypeT.ByU16) && (inputSize >= Limit64k)) { return(0); } if (inputSize < LZ4MinLength) { goto _last_literals; } PutPosition(ip, cctx->HashTable, tableType, ibase); ip++; uint forwardH = HashPosition(ip, tableType); for (; ;) { long refDelta = 0L; byte *match; byte *token; { byte *forwardIp = ip; uint step = 1u; uint searchMatchNb = acceleration << SkipTrigger; do { uint h = forwardH; ip = forwardIp; forwardIp += step; step = searchMatchNb++ >> SkipTrigger; if (forwardIp > mflimit) { goto _last_literals; } match = GetPositionOnHash(h, cctx->HashTable, tableType, ibase); if (dict == DictDirective.UsingExtDict) { if (match < source) { refDelta = dictDelta; lowLimit = dictionary; } else { refDelta = 0; lowLimit = source; } } forwardH = HashPosition(forwardIp, tableType); PutPositionOnHash(ip, h, cctx->HashTable, tableType, ibase); }while ((dictIssue == DictIssueDirective.DictSmall) && (match < lowRefLimit) || (tableType != TableTypeT.ByU16) && (match + MaxDistance < ip) || (LZ4MemoryHelper.Peek32(match + refDelta) != LZ4MemoryHelper.Peek32(ip))); } while ((ip > anchor) && (match + refDelta > lowLimit) && (ip[-1] == match[refDelta - 1])) { ip--; match--; } { uint litLength = (uint)(ip - anchor); token = op++; if ((outputLimited == LimitedOutputDirective.LimitedOutput) && (op + litLength + (2 + 1 + LastLiterals) + litLength / 255 > olimit)) { return(0); } if (litLength >= RunMask) { int len = (int)(litLength - RunMask); *token = (byte)(RunMask << MLBits); for (; len >= 255; len -= 255) { *op++ = 255; } *op++ = (byte)len; } else { *token = (byte)(litLength << MLBits); } LZ4MemoryHelper.WildCopy(op, anchor, op + litLength); op += litLength; } _next_match: LZ4MemoryHelper.Poke16(op, (ushort)(ip - match)); op += 2; { uint matchCode; if ((dict == DictDirective.UsingExtDict) && (lowLimit == dictionary)) { match += refDelta; byte *limit = ip + (dictEnd - match); if (limit > matchlimit) { limit = matchlimit; } matchCode = Count(ip + MinMatch, match + MinMatch, limit); ip += MinMatch + matchCode; if (ip == limit) { uint more = Count(ip, source, matchlimit); matchCode += more; ip += more; } } else { matchCode = Count(ip + MinMatch, match + MinMatch, matchlimit); ip += MinMatch + matchCode; } if ((outputLimited == LimitedOutputDirective.LimitedOutput) && (op + (1 + LastLiterals) + (matchCode >> 8) > olimit)) { return(0); } if (matchCode >= MLMask) { *token += (byte)MLMask; matchCode -= MLMask; LZ4MemoryHelper.Poke32(op, 0xFFFFFFFF); while (matchCode >= 4 * 255) { op += 4; LZ4MemoryHelper.Poke32(op, 0xFFFFFFFF); matchCode -= 4 * 255; } op += matchCode / 255; *op++ = (byte)(matchCode % 255); } else { *token += (byte)matchCode; } } anchor = ip; if (ip > mflimit) { break; } PutPosition(ip - 2, cctx->HashTable, tableType, ibase); match = GetPosition(ip, cctx->HashTable, tableType, ibase); if (dict == DictDirective.UsingExtDict) { if (match < source) { refDelta = dictDelta; lowLimit = dictionary; } else { refDelta = 0; lowLimit = source; } } PutPosition(ip, cctx->HashTable, tableType, ibase); if ((dictIssue != DictIssueDirective.DictSmall || match >= lowRefLimit) && (match + MaxDistance >= ip) && (LZ4MemoryHelper.Peek32(match + refDelta) == LZ4MemoryHelper.Peek32(ip))) { token = op++; *token = 0; goto _next_match; } forwardH = HashPosition(++ip, tableType); } _last_literals: { int lastRun = (int)(iend - anchor); if ((outputLimited == LimitedOutputDirective.LimitedOutput) && (op - dest + lastRun + 1 + (lastRun + 255 - RunMask) / 255 > (uint)maxOutputSize)) { return(0); } if (lastRun >= RunMask) { int accumulator = (int)(lastRun - RunMask); *op++ = (byte)(RunMask << MLBits); for (; accumulator >= 255; accumulator -= 255) { *op++ = 255; } *op++ = (byte)accumulator; } else { *op++ = (byte)(lastRun << MLBits); } LZ4MemoryHelper.Copy(op, anchor, lastRun); op += lastRun; } return((int)(op - dest)); }
private static int CompressHashChain(CCtxT *ctx, byte *source, byte *dest, int *srcSizePtr, int maxOutputSize, uint maxNbAttempts, LimitedOutputDirective limit) { int inputSize = *srcSizePtr; int patternAnalysis = maxNbAttempts > 64 ? 1 : 0; // levels 8+ byte *ip = source; byte *anchor = ip; byte *iend = ip + inputSize; byte *mflimit = iend - MFLimit; byte *matchlimit = (iend - LastLiterals); byte *optr; byte *op = dest; byte *oend = op + maxOutputSize; byte *refPos = null; byte *start2 = null; byte *ref2 = null; byte *start3 = null; byte *ref3 = null; // init *srcSizePtr = 0; if (limit == LimitedOutputDirective.LimitedDestSize) { oend -= LastLiterals; // Hack for support LZ4 format restriction } if (inputSize < LZ4MinLength) { goto _last_literals; // Input too small, no compression (all literals) } // Main Loop while (ip < mflimit) { int ml = InsertAndFindBestMatch( ctx, ip, matchlimit, &refPos, (int)maxNbAttempts, patternAnalysis); if (ml < MinMatch) { ip++; continue; } // saved, in case we would skip too much byte *start0 = ip; byte *ref0 = refPos; int ml0 = ml; _Search2: int ml2; if (ip + ml < mflimit) { ml2 = InsertAndGetWiderMatch( ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, (int)maxNbAttempts, patternAnalysis); } else { ml2 = ml; } if (ml2 == ml) { // No better match optr = op; if (EncodeSequence(&ip, &op, &anchor, ml, refPos, limit, oend) != 0) { goto _dest_overflow; } continue; } if (start0 < ip) { if (start2 < ip + ml0) { // empirical ip = start0; refPos = ref0; ml = ml0; } } // Here, start0==ip if (start2 - ip < 3) { // First Match too small : removed ml = ml2; ip = start2; refPos = ref2; goto _Search2; } _Search3: // At this stage, we have : // ml2 > ml1, and // ip1+3 <= ip2 (usually < ip1+ml1) if ((start2 - ip) < OptimalML) { int newMl = ml; if (newMl > OptimalML) { newMl = OptimalML; } if (ip + newMl > start2 + ml2 - MinMatch) { newMl = (int)(start2 - ip) + ml2 - MinMatch; } int correction = newMl - (int)(start2 - ip); if (correction > 0) { start2 += correction; ref2 += correction; ml2 -= correction; } } // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) int ml3; if (start2 + ml2 < mflimit) { ml3 = InsertAndGetWiderMatch( ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, (int)maxNbAttempts, patternAnalysis); } else { ml3 = ml2; } if (ml3 == ml2) { // No better match : 2 sequences to encode // ip & ref are known; Now for ml if (start2 < ip + ml) { ml = (int)(start2 - ip); } // Now, encode 2 sequences optr = op; if (EncodeSequence(&ip, &op, &anchor, ml, refPos, limit, oend) != 0) { goto _dest_overflow; } ip = start2; optr = op; if (EncodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend) != 0) { goto _dest_overflow; } continue; } if (start3 < ip + ml + 3) { // Not enough space for match 2 : remove it if (start3 >= (ip + ml)) { // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 if (start2 < ip + ml) { int correction = (int)(ip + ml - start2); start2 += correction; ref2 += correction; ml2 -= correction; if (ml2 < MinMatch) { start2 = start3; ref2 = ref3; ml2 = ml3; } } optr = op; if (EncodeSequence(&ip, &op, &anchor, ml, refPos, limit, oend) != 0) { goto _dest_overflow; } ip = start3; refPos = ref3; ml = ml3; start0 = start2; ref0 = ref2; ml0 = ml2; goto _Search2; } start2 = start3; ref2 = ref3; ml2 = ml3; goto _Search3; } // OK, now we have 3 ascending matches; let's write at least the first one // ip & ref are known; Now for ml if (start2 < ip + ml) { if (start2 - ip < (int)MLMask) { if (ml > OptimalML) { ml = OptimalML; } if (ip + ml > start2 + ml2 - MinMatch) { ml = (int)(start2 - ip) + ml2 - MinMatch; } int correction = ml - (int)(start2 - ip); if (correction > 0) { start2 += correction; ref2 += correction; ml2 -= correction; } } else { ml = (int)(start2 - ip); } } optr = op; if (EncodeSequence(&ip, &op, &anchor, ml, refPos, limit, oend) != 0) { goto _dest_overflow; } ip = start2; refPos = ref2; ml = ml2; start2 = start3; ref2 = ref3; ml2 = ml3; goto _Search3; } _last_literals: // Encode last literals { size_t lastRunSize = (size_t)(iend - anchor); // literals size_t litLength = (lastRunSize + 255 - RunMask) / 255; size_t totalSize = 1 + litLength + lastRunSize; if (limit == LimitedOutputDirective.LimitedDestSize) { oend += LastLiterals; // restore correct value } if ((limit != 0) && (op + totalSize > oend)) { if (limit == LimitedOutputDirective.LimitedOutput) { return(0); // Check output limit } // adapt lastRunSize to fill 'dest' lastRunSize = (size_t)(oend - op) - 1; litLength = (lastRunSize + 255 - RunMask) / 255; lastRunSize -= litLength; } ip = anchor + lastRunSize; if (lastRunSize >= RunMask) { size_t accumulator = lastRunSize - RunMask; *op++ = (byte)(RunMask << MLBits); for (; accumulator >= 255; accumulator -= 255) { *op++ = 255; } *op++ = (byte)accumulator; } else { *op++ = (byte)(lastRunSize << MLBits); } LZ4MemoryHelper.Copy(op, anchor, (int)lastRunSize); op += lastRunSize; } // End *srcSizePtr = (int)(ip - source); return((int)(op - dest)); _dest_overflow: if (limit != LimitedOutputDirective.LimitedDestSize) { return(0); } op = optr; // restore correct out pointer goto _last_literals; }
// ----------------------------------------------------------------- private static int CompressOptimal(CCtxT *ctx, byte *source, byte *dst, int *srcSizePtr, int dstCapacity, int nbSearches, size_t sufficientLen, LimitedOutputDirective limit, int fullUpdate) { const int TrailingLiterals = 3; OptimalT *opt = stackalloc OptimalT[OptNum + TrailingLiterals]; byte *ip = source; byte *anchor = ip; byte *iend = ip + *srcSizePtr; byte *mflimit = iend - MFLimit; byte *matchlimit = iend - LastLiterals; byte *op = dst; byte *opSaved; byte *oend = op + dstCapacity; *srcSizePtr = 0; if (limit == LimitedOutputDirective.LimitedDestSize) { oend -= LastLiterals; } if (sufficientLen >= OptNum) { sufficientLen = OptNum - 1; } // Main Loop while (ip < mflimit) { int llen = (int)(ip - anchor); int bestMlen, bestOff; int cur, lastMatchPos; MatchT firstMatch = FindLongerMatch(ctx, ip, matchlimit, MinMatch - 1, nbSearches); if (firstMatch.Len == 0) { ip++; continue; } if ((size_t)firstMatch.Len > sufficientLen) { int firstML = firstMatch.Len; byte *matchPos = ip - firstMatch.Off; opSaved = op; if (EncodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) != 0) { goto _dest_overflow; } continue; } // set prices for first positions (literals) { int rPos; for (rPos = 0; rPos < MinMatch; rPos++) { int cost = LiteralsPrice(llen + rPos); opt[rPos].Mlen = 1; opt[rPos].Off = 0; opt[rPos].Litlen = llen + rPos; opt[rPos].Price = cost; } } // set prices using initial match { int mlen = MinMatch; int matchML = firstMatch.Len; // necessarily < sufficient_len < LZ4_OPT_NUM int offset = firstMatch.Off; for (; mlen <= matchML; mlen++) { int cost = SequencePrice(llen, mlen); opt[mlen].Mlen = mlen; opt[mlen].Off = offset; opt[mlen].Litlen = llen; opt[mlen].Price = cost; } } lastMatchPos = firstMatch.Len; { int addLit; for (addLit = 1; addLit <= TrailingLiterals; addLit++) { opt[lastMatchPos + addLit].Mlen = 1; opt[lastMatchPos + addLit].Off = 0; opt[lastMatchPos + addLit].Litlen = addLit; opt[lastMatchPos + addLit].Price = opt[lastMatchPos].Price + LiteralsPrice(addLit); } } // check further positions for (cur = 1; cur < lastMatchPos; cur++) { byte *curPtr = ip + cur; if (curPtr >= mflimit) { break; } if (fullUpdate != 0) { if ((opt[cur + 1].Price <= opt[cur].Price) && (opt[cur + MinMatch].Price < opt[cur].Price + 3)) { continue; } } else { if (opt[cur + 1].Price <= opt[cur].Price) { continue; } } MatchT newMatch = FindLongerMatch( ctx, curPtr, matchlimit, fullUpdate != 0 ? MinMatch - 1 : lastMatchPos - cur, nbSearches); if (newMatch.Len == 0) { continue; } if ((size_t)newMatch.Len > sufficientLen || newMatch.Len + cur >= OptNum) { // immediate encoding bestMlen = newMatch.Len; bestOff = newMatch.Off; lastMatchPos = cur + 1; goto encode; } // before match : set price with literals at beginning { int baseLitlen = opt[cur].Litlen; for (int litlen = 1; litlen < MinMatch; litlen++) { int price = opt[cur].Price - LiteralsPrice(baseLitlen) + LiteralsPrice(baseLitlen + litlen); int pos = cur + litlen; if (price >= opt[pos].Price) { continue; } opt[pos].Mlen = 1; opt[pos].Off = 0; opt[pos].Litlen = baseLitlen + litlen; opt[pos].Price = price; } } // set prices using match at position = cur { int matchML = newMatch.Len; int ml = MinMatch; for (; ml <= matchML; ml++) { int pos = cur + ml; int offset = newMatch.Off; int price; int ll; if (opt[cur].Mlen == 1) { ll = opt[cur].Litlen; price = ((cur > ll) ? opt[cur - ll].Price : 0) + SequencePrice(ll, ml); } else { ll = 0; price = opt[cur].Price + SequencePrice(0, ml); } if ((pos > lastMatchPos + TrailingLiterals) || (price <= opt[pos].Price)) { if (ml == matchML && lastMatchPos < pos) { lastMatchPos = pos; } opt[pos].Mlen = ml; opt[pos].Off = offset; opt[pos].Litlen = ll; opt[pos].Price = price; } } } { int addLit; for (addLit = 1; addLit <= TrailingLiterals; addLit++) { opt[lastMatchPos + addLit].Mlen = 1; opt[lastMatchPos + addLit].Off = 0; opt[lastMatchPos + addLit].Litlen = addLit; opt[lastMatchPos + addLit].Price = opt[lastMatchPos].Price + LiteralsPrice(addLit); } } } bestMlen = opt[lastMatchPos].Mlen; bestOff = opt[lastMatchPos].Off; cur = lastMatchPos - bestMlen; encode: // cur, last_match_pos, best_mlen, best_off must be set { int candidate_pos = cur; int selected_matchLength = bestMlen; int selected_offset = bestOff; while (true) { int next_matchLength = opt[candidate_pos].Mlen; int next_offset = opt[candidate_pos].Off; opt[candidate_pos].Mlen = selected_matchLength; opt[candidate_pos].Off = selected_offset; selected_matchLength = next_matchLength; selected_offset = next_offset; if (next_matchLength > candidate_pos) { break; } candidate_pos -= next_matchLength; } } { int rPos = 0; while (rPos < lastMatchPos) { int ml = opt[rPos].Mlen; int offset = opt[rPos].Off; if (ml == 1) { ip++; rPos++; continue; } rPos += ml; opSaved = op; if (EncodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) != 0) { goto _dest_overflow; } } } } _last_literals: { size_t lastRunSize = (size_t)(iend - anchor); size_t litLength = (lastRunSize + 255 - RunMask) / 255; size_t totalSize = 1 + litLength + lastRunSize; if (limit == LimitedOutputDirective.LimitedDestSize) { oend += LastLiterals; } if (limit != 0 && op + totalSize > oend) { if (limit == LimitedOutputDirective.LimitedOutput) { return(0); } lastRunSize = (size_t)(oend - op) - 1; litLength = (lastRunSize + 255 - RunMask) / 255; lastRunSize -= litLength; } ip = anchor + lastRunSize; if (lastRunSize >= RunMask) { size_t accumulator = lastRunSize - RunMask; * op++ = (byte)(RunMask << MLBits); for (; accumulator >= 255; accumulator -= 255) { *op++ = 255; } *op++ = (byte)accumulator; } else { *op++ = (byte)(lastRunSize << MLBits); } LZ4MemoryHelper.Copy(op, anchor, (int)lastRunSize); op += lastRunSize; } // End *srcSizePtr = (int)(ip - source); return((int)(op - dst)); _dest_overflow: if (limit != LimitedOutputDirective.LimitedDestSize) { return(0); } op = opSaved; // restore correct out pointer goto _last_literals; }
private static int EncodeSequence(byte **ip, byte **op, byte **anchor, int matchLength, byte *match, LimitedOutputDirective limit, byte *oend) { byte *token = (*op)++; size_t length = (size_t)(*ip - *anchor); if ((limit != LimitedOutputDirective.NoLimit) && (*op + (length >> 8) + length + (2 + 1 + LastLiterals) > oend)) { return(1); } if (length >= RunMask) { size_t len = length - RunMask; * token = (byte)(RunMask << MLBits); for (; len >= 255; len -= 255) { *(*op)++ = 255; } *(*op)++ = (byte)len; } else { *token = (byte)(length << MLBits); } LZ4MemoryHelper.WildCopy(*op, *anchor, (*op) + length); *op += length; LZ4MemoryHelper.Poke16(*op, (ushort)(*ip - match)); *op += 2; length = (size_t)(matchLength - MinMatch); if ((limit != LimitedOutputDirective.NoLimit) && (*op + (length >> 8) + (1 + LastLiterals) > oend)) { return(1); } if (length >= MLMask) { *token += (byte)MLMask; length -= MLMask; for (; length >= 510; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; } if (length >= 255) { length -= 255; *(*op)++ = 255; } *(*op)++ = (byte)length; } else { *token += (byte)(length); } *ip += matchLength; *anchor = *ip; return(0); }
private static int CompressHCContinueGeneric(CCtxT *ctxPtr, byte *src, byte *dst, int *srcSizePtr, int dstCapacity, LimitedOutputDirective limit) { // auto-init if forgotten if (ctxPtr->BaseP == null) { InitializeHC(ctxPtr, src); } // Check overflow if ((size_t)(ctxPtr->End - ctxPtr->BaseP) > 2 * GB) { size_t dictSize = (size_t)(ctxPtr->End - ctxPtr->BaseP) - ctxPtr->DictLimit; if (dictSize > 64 * KB) { dictSize = 64 * KB; } LoadDictHC(ctxPtr, ctxPtr->End - dictSize, (int)dictSize); } // Check if blocks follow each other if (src != ctxPtr->End) { SetExternalDict(ctxPtr, src); } // Check overlapping input/dictionary space { byte *sourceEnd = src + *srcSizePtr; byte *dictBegin = ctxPtr->DictBase + ctxPtr->LowLimit; byte *dictEnd = ctxPtr->DictBase + ctxPtr->DictLimit; if (sourceEnd > dictBegin && src < dictEnd) { if (sourceEnd > dictEnd) { sourceEnd = dictEnd; } ctxPtr->LowLimit = (uint)(sourceEnd - ctxPtr->DictBase); if (ctxPtr->DictLimit - ctxPtr->LowLimit < 4) { ctxPtr->LowLimit = ctxPtr->DictLimit; } } } return(CompressGeneric( ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->CompressionLevel, limit)); }
private static int CompressGeneric(CCtxT *ctx, byte *src, byte *dst, int *srcSizePtr, int dstCapacity, int cLevel, LimitedOutputDirective limit) { if ((limit == LimitedOutputDirective.LimitedDestSize) && (dstCapacity < 1)) { return(0); // Impossible to store anything } if (*srcSizePtr > MaxInputSize) { return(0); // Unsupported input size (too large or negative) } ctx->End += *srcSizePtr; if (cLevel < 1) { cLevel = CLevelDefault; // note : convention is different from lz4frame, maybe something to review } cLevel = Math.Min(CLevelMax, cLevel); CParams cParam = ClTable[cLevel]; if (cParam.Strat == StratE.HC) { return(CompressHashChain( ctx, src, dst, srcSizePtr, dstCapacity, cParam.NbSearches, limit)); } return(CompressOptimal( ctx, src, dst, srcSizePtr, dstCapacity, (int)cParam.NbSearches, cParam.TargetLength, limit, cLevel == CLevelMax ? 1 : 0)); // ultra mode }