Example #1
0
        private static int CompressDestSizeGeneric(StreamT *ctx, byte *src, byte *dst, int *srcSizePtr, int targetDstSize, TableTypeT tableType)
        {
            byte *ip         = src;
            byte *srcBase    = src;
            byte *lowLimit   = src;
            byte *anchor     = ip;
            byte *iend       = ip + *srcSizePtr;
            byte *mflimit    = iend - MFLimit;
            byte *matchlimit = iend - LastLiterals;

            byte *op        = dst;
            byte *oend      = op + targetDstSize;
            byte *oMaxLit   = op + targetDstSize - 2 - 8 - 1;
            byte *oMaxMatch = op + targetDstSize - (LastLiterals + 1);
            byte *oMaxSeq   = oMaxLit - 1;

            if (targetDstSize < 1)
            {
                return(0);
            }
            if (*srcSizePtr > MaxInputSize)
            {
                return(0);
            }
            if ((tableType == TableTypeT.ByU16) && (*srcSizePtr >= Limit64k))
            {
                return(0);
            }

            if (*srcSizePtr < LZ4MinLength)
            {
                goto _last_literals; // Input too small, no compression (all literals)
            }
            *srcSizePtr = 0;
            PutPosition(ip, ctx->HashTable, tableType, srcBase);
            ip++;
            uint forwardH = HashPosition(ip, tableType);

            for (; ;)
            {
                byte *match;
                byte *token;

                {
                    byte *forwardIp     = ip;
                    uint  step          = 1u;
                    uint  searchMatchNb = 1u << SkipTrigger;

                    do
                    {
                        uint h = forwardH;
                        ip         = forwardIp;
                        forwardIp += step;
                        step       = searchMatchNb++ >> SkipTrigger;

                        if (forwardIp > mflimit)
                        {
                            goto _last_literals;
                        }

                        match    = GetPositionOnHash(h, ctx->HashTable, tableType, srcBase);
                        forwardH = HashPosition(forwardIp, tableType);
                        PutPositionOnHash(ip, h, ctx->HashTable, tableType, srcBase);
                    }while ((tableType != TableTypeT.ByU16) && (match + MaxDistance < ip) ||
                            (LZ4MemoryHelper.Peek32(match) != LZ4MemoryHelper.Peek32(ip)));
                }

                while ((ip > anchor) && (match > lowLimit) && (ip[-1] == match[-1]))
                {
                    ip--;
                    match--;
                }

                {
                    uint litLength = (uint)(ip - anchor);
                    token = op++;
                    if (op + (litLength + 240) / 255 + litLength > oMaxLit)
                    {
                        op--;
                        goto _last_literals;
                    }

                    if (litLength >= RunMask)
                    {
                        uint len = litLength - RunMask;

                        *token = (byte)(RunMask << MLBits);
                        for (; len >= 255; len -= 255)
                        {
                            *op++ = 255;
                        }

                        *op++ = (byte)len;
                    }
                    else
                    {
                        *token = (byte)(litLength << MLBits);
                    }

                    LZ4MemoryHelper.WildCopy(op, anchor, op + litLength);
                    op += litLength;
                }

_next_match:
                LZ4MemoryHelper.Poke16(op, (ushort)(ip - match));
                op += 2;

                {
                    int matchLength = (int)Count(ip + MinMatch, match + MinMatch, matchlimit);

                    if (op + (matchLength + 240) / 255 > oMaxMatch)
                    {
                        matchLength = (int)(15 - 1 + (oMaxMatch - op) * 255);
                    }

                    ip += MinMatch + matchLength;

                    if (matchLength >= MLMask)
                    {
                        *token += (byte)MLMask;
                        matchLength -= (int)MLMask;
                        while (matchLength >= 255)
                        {
                            matchLength -= 255;
                            *op++ = 255;
                        }

                        *op++ = (byte)matchLength;
                    }
                    else
                    {
                        *token += (byte)matchLength;
                    }
                }

                anchor = ip;

                if (ip > mflimit)
                {
                    break;
                }
                if (op > oMaxSeq)
                {
                    break;
                }

                PutPosition(ip - 2, ctx->HashTable, tableType, srcBase);

                match = GetPosition(ip, ctx->HashTable, tableType, srcBase);
                PutPosition(ip, ctx->HashTable, tableType, srcBase);
                if ((match + MaxDistance >= ip) && (LZ4MemoryHelper.Peek32(match) == LZ4MemoryHelper.Peek32(ip)))
                {
                    token = op++;
                    *token = 0;
                    goto _next_match;
                }

                forwardH = HashPosition(++ip, tableType);
            }

_last_literals:
            {
                int lastRunSize = (int)(iend - anchor);
                if (op + 1 + (lastRunSize + 240) / 255 + lastRunSize > oend)
                {
                    lastRunSize  = (int)(oend - op) - 1;
                    lastRunSize -= (lastRunSize + 240) / 255;
                }

                ip = anchor + lastRunSize;

                if (lastRunSize >= RunMask)
                {
                    long accumulator = lastRunSize - RunMask;

                    *op++ = (byte)(RunMask << MLBits);
                    for (; accumulator >= 255; accumulator -= 255)
                    {
                        *op++ = 255;
                    }

                    *op++ = (byte)accumulator;
                }
                else
                {
                    *op++ = (byte)(lastRunSize << MLBits);
                }

                LZ4MemoryHelper.Copy(op, anchor, lastRunSize);
                op += lastRunSize;
            }

            *srcSizePtr = (int)(ip - src);
            return((int)(op - dst));
        }
Example #2
0
        public static int DecompressGeneric(byte *src, byte *dst, int srcSize, int outputSize,
                                            EndConditionDirective endOnInput, EarlyEndDirective partialDecoding,
                                            int targetOutputSize, DictDirective dict, byte *lowPrefix, byte *dictStart, int dictSize)
        {
            byte *ip   = src;
            byte *iend = ip + srcSize;

            byte *op    = dst;
            byte *oend  = op + outputSize;
            byte *oexit = op + targetOutputSize;

            byte *dictEnd = dictStart + dictSize;

            bool safeDecode  = endOnInput == EndConditionDirective.EndOnInputSize;
            bool checkOffset = safeDecode && dictSize < 64 * KB;

            if ((partialDecoding != EarlyEndDirective.Full) && (oexit > oend - MFLimit))
            {
                oexit = oend - MFLimit;
            }
            if ((endOnInput == EndConditionDirective.EndOnInputSize) && (outputSize == 0))
            {
                return(srcSize == 1 && *ip == 0 ? 0 : -1);
            }
            if ((endOnInput != EndConditionDirective.EndOnInputSize) && (outputSize == 0))
            {
                return(*ip == 0 ? 1 : -1);
            }

            for (; ;)
            {
                int  length;
                uint token = *ip++;

                if ((ip + 14 + 2 <= iend) &&
                    (op + 14 + 18 <= oend) &&
                    (token < 15 << MLBits) &&
                    ((token & MLMask) != 15))
                {
                    int   ll       = (int)(token >> MLBits);
                    int   off      = LZ4MemoryHelper.Peek16(ip + ll);
                    byte *matchPtr = op + ll - off;
                    if (off >= 18 && matchPtr >= lowPrefix)
                    {
                        int ml = (int)((token & MLMask) + MinMatch);
                        LZ4MemoryHelper.Copy16(op, ip);
                        op += ll;
                        ip += ll + 2;
                        LZ4MemoryHelper.Copy18(op, matchPtr);
                        op += ml;
                        continue;
                    }
                }

                if ((length = (int)(token >> MLBits)) == RunMask)
                {
                    uint s;
                    do
                    {
                        s       = *ip++;
                        length += (int)s;
                    }while (
                        (endOnInput != EndConditionDirective.EndOnInputSize || ip < iend - RunMask) &&
                        (s == 255));

                    if (safeDecode && op + length < op)
                    {
                        goto _output_error;
                    }
                    if (safeDecode && ip + length < ip)
                    {
                        goto _output_error;
                    }
                }

                byte *cpy = op + length;
                if ((endOnInput == EndConditionDirective.EndOnInputSize) &&
                    ((cpy > (partialDecoding == EarlyEndDirective.Partial ? oexit : oend - MFLimit)) ||
                     (ip + length > iend - (2 + 1 + LastLiterals))) ||
                    (endOnInput != EndConditionDirective.EndOnInputSize) && (cpy > oend - WildCopyLength))
                {
                    if (partialDecoding == EarlyEndDirective.Partial)
                    {
                        if (cpy > oend)
                        {
                            goto _output_error;
                        }
                        if ((endOnInput == EndConditionDirective.EndOnInputSize) && (ip + length > iend))
                        {
                            goto _output_error;
                        }
                    }
                    else
                    {
                        if ((endOnInput != EndConditionDirective.EndOnInputSize) && (cpy != oend))
                        {
                            goto _output_error;
                        }

                        if ((endOnInput == EndConditionDirective.EndOnInputSize) &&
                            (ip + length != iend || cpy > oend))
                        {
                            goto _output_error;
                        }
                    }

                    LZ4MemoryHelper.Copy(op, ip, length);
                    ip += length;
                    op += length;
                    break;
                }

                LZ4MemoryHelper.WildCopy(op, ip, cpy);
                ip += length;
                op  = cpy;

                int offset = LZ4MemoryHelper.Peek16(ip);
                ip += 2;
                byte *match = op - offset;
                if (checkOffset && match + dictSize < lowPrefix)
                {
                    goto _output_error;
                }

                LZ4MemoryHelper.Poke32(op, (uint)offset);

                length = (int)(token & MLMask);
                if (length == MLMask)
                {
                    uint s;
                    do
                    {
                        s = *ip++;
                        if ((endOnInput == EndConditionDirective.EndOnInputSize) && (ip > iend - LastLiterals))
                        {
                            goto _output_error;
                        }

                        length += (int)s;
                    }while (s == 255);

                    if (safeDecode && (op + length < op))
                    {
                        goto _output_error;
                    }
                }

                length += MinMatch;

                if ((dict == DictDirective.UsingExtDict) && (match < lowPrefix))
                {
                    if (op + length > oend - LastLiterals)
                    {
                        goto _output_error;
                    }

                    if (length <= lowPrefix - match)
                    {
                        LZ4MemoryHelper.Move(op, dictEnd - (lowPrefix - match), length);
                        op += length;
                    }
                    else
                    {
                        int copySize = (int)(lowPrefix - match);
                        int restSize = length - copySize;
                        LZ4MemoryHelper.Copy(op, dictEnd - copySize, copySize);
                        op += copySize;
                        if (restSize > (int)(op - lowPrefix))
                        {
                            byte *endOfMatch = op + restSize;
                            byte *copyFrom   = lowPrefix;
                            while (op < endOfMatch)
                            {
                                *op++ = *copyFrom++;
                            }
                        }
                        else
                        {
                            LZ4MemoryHelper.Copy(op, lowPrefix, restSize);
                            op += restSize;
                        }
                    }

                    continue;
                }

                cpy = op + length;
                if (offset < 8)
                {
                    op[0]  = match[0];
                    op[1]  = match[1];
                    op[2]  = match[2];
                    op[3]  = match[3];
                    match += _inc32table[offset];
                    LZ4MemoryHelper.Copy(op + 4, match, 4);
                    match -= _dec64table[offset];
                }
                else
                {
                    LZ4MemoryHelper.Copy8(op, match);
                    match += 8;
                }

                op += 8;

                if (cpy > oend - 12)
                {
                    byte *oCopyLimit = oend - (WildCopyLength - 1);
                    if (cpy > oend - LastLiterals)
                    {
                        goto _output_error;
                    }

                    if (op < oCopyLimit)
                    {
                        LZ4MemoryHelper.WildCopy(op, match, oCopyLimit);
                        match += oCopyLimit - op;
                        op     = oCopyLimit;
                    }

                    while (op < cpy)
                    {
                        *op++ = *match++;
                    }
                }
                else
                {
                    LZ4MemoryHelper.Copy8(op, match);
                    if (length > 16)
                    {
                        LZ4MemoryHelper.WildCopy(op + 8, match + 8, cpy);
                    }
                }

                op = cpy; // correction
            }

            // end of decoding
            if (endOnInput == EndConditionDirective.EndOnInputSize)
            {
                return((int)(op - dst)); // Nb of output bytes decoded
            }
            return((int)(ip - src));     // Nb of input bytes read

            // Overflow error detected
_output_error:
            return((int)-(ip - src) - 1);
        }
Example #3
0
        public static int CompressGeneric(StreamT *cctx, byte *source, byte *dest, int inputSize, int maxOutputSize,
                                          LimitedOutputDirective outputLimited, TableTypeT tableType,
                                          DictDirective dict, DictIssueDirective dictIssue, uint acceleration)
        {
            byte *ip = source;
            byte *ibase;
            byte *lowLimit;
            byte *lowRefLimit = ip - cctx->DictSize;
            byte *dictionary  = cctx->Dictionary;
            byte *dictEnd     = dictionary + cctx->DictSize;
            long  dictDelta   = dictEnd - source;
            byte *anchor      = source;
            byte *iend        = ip + inputSize;
            byte *mflimit     = iend - MFLimit;
            byte *matchlimit  = iend - LastLiterals;

            byte *op     = dest;
            byte *olimit = op + maxOutputSize;

            if (inputSize > MaxInputSize)
            {
                return(0);
            }

            switch (dict)
            {
            case DictDirective.WithPrefix64k:
                ibase    = source - cctx->CurrentOffset;
                lowLimit = source - cctx->DictSize;
                break;

            case DictDirective.UsingExtDict:
                ibase    = source - cctx->CurrentOffset;
                lowLimit = source;
                break;

            default:
                ibase    = source;
                lowLimit = source;
                break;
            }

            if ((tableType == TableTypeT.ByU16) && (inputSize >= Limit64k))
            {
                return(0);
            }

            if (inputSize < LZ4MinLength)
            {
                goto _last_literals;
            }

            PutPosition(ip, cctx->HashTable, tableType, ibase);
            ip++;
            uint forwardH = HashPosition(ip, tableType);

            for (; ;)
            {
                long  refDelta = 0L;
                byte *match;
                byte *token;

                {
                    byte *forwardIp     = ip;
                    uint  step          = 1u;
                    uint  searchMatchNb = acceleration << SkipTrigger;
                    do
                    {
                        uint h = forwardH;
                        ip         = forwardIp;
                        forwardIp += step;
                        step       = searchMatchNb++ >> SkipTrigger;

                        if (forwardIp > mflimit)
                        {
                            goto _last_literals;
                        }

                        match = GetPositionOnHash(h, cctx->HashTable, tableType, ibase);
                        if (dict == DictDirective.UsingExtDict)
                        {
                            if (match < source)
                            {
                                refDelta = dictDelta;
                                lowLimit = dictionary;
                            }
                            else
                            {
                                refDelta = 0;
                                lowLimit = source;
                            }
                        }

                        forwardH = HashPosition(forwardIp, tableType);
                        PutPositionOnHash(ip, h, cctx->HashTable, tableType, ibase);
                    }while ((dictIssue == DictIssueDirective.DictSmall) && (match < lowRefLimit) ||
                            (tableType != TableTypeT.ByU16) && (match + MaxDistance < ip) ||
                            (LZ4MemoryHelper.Peek32(match + refDelta) != LZ4MemoryHelper.Peek32(ip)));
                }

                while ((ip > anchor) && (match + refDelta > lowLimit) && (ip[-1] == match[refDelta - 1]))
                {
                    ip--;
                    match--;
                }

                {
                    uint litLength = (uint)(ip - anchor);
                    token = op++;
                    if ((outputLimited == LimitedOutputDirective.LimitedOutput) &&
                        (op + litLength + (2 + 1 + LastLiterals) + litLength / 255 > olimit))
                    {
                        return(0);
                    }

                    if (litLength >= RunMask)
                    {
                        int len = (int)(litLength - RunMask);

                        *token = (byte)(RunMask << MLBits);
                        for (; len >= 255; len -= 255)
                        {
                            *op++ = 255;
                        }

                        *op++ = (byte)len;
                    }
                    else
                    {
                        *token = (byte)(litLength << MLBits);
                    }

                    LZ4MemoryHelper.WildCopy(op, anchor, op + litLength);
                    op += litLength;
                }

_next_match:
                LZ4MemoryHelper.Poke16(op, (ushort)(ip - match));
                op += 2;

                {
                    uint matchCode;

                    if ((dict == DictDirective.UsingExtDict) && (lowLimit == dictionary))
                    {
                        match += refDelta;
                        byte *limit = ip + (dictEnd - match);
                        if (limit > matchlimit)
                        {
                            limit = matchlimit;
                        }
                        matchCode = Count(ip + MinMatch, match + MinMatch, limit);
                        ip       += MinMatch + matchCode;
                        if (ip == limit)
                        {
                            uint more = Count(ip, source, matchlimit);
                            matchCode += more;
                            ip        += more;
                        }
                    }
                    else
                    {
                        matchCode = Count(ip + MinMatch, match + MinMatch, matchlimit);
                        ip       += MinMatch + matchCode;
                    }

                    if ((outputLimited == LimitedOutputDirective.LimitedOutput) &&
                        (op + (1 + LastLiterals) + (matchCode >> 8) > olimit))
                    {
                        return(0);
                    }

                    if (matchCode >= MLMask)
                    {
                        *token += (byte)MLMask;
                        matchCode -= MLMask;
                        LZ4MemoryHelper.Poke32(op, 0xFFFFFFFF);
                        while (matchCode >= 4 * 255)
                        {
                            op += 4;
                            LZ4MemoryHelper.Poke32(op, 0xFFFFFFFF);
                            matchCode -= 4 * 255;
                        }

                        op += matchCode / 255;

                        *op++ = (byte)(matchCode % 255);
                    }
                    else
                    {
                        *token += (byte)matchCode;
                    }
                }

                anchor = ip;

                if (ip > mflimit)
                {
                    break;
                }

                PutPosition(ip - 2, cctx->HashTable, tableType, ibase);

                match = GetPosition(ip, cctx->HashTable, tableType, ibase);
                if (dict == DictDirective.UsingExtDict)
                {
                    if (match < source)
                    {
                        refDelta = dictDelta;
                        lowLimit = dictionary;
                    }
                    else
                    {
                        refDelta = 0;
                        lowLimit = source;
                    }
                }

                PutPosition(ip, cctx->HashTable, tableType, ibase);
                if ((dictIssue != DictIssueDirective.DictSmall || match >= lowRefLimit) &&
                    (match + MaxDistance >= ip) &&
                    (LZ4MemoryHelper.Peek32(match + refDelta) == LZ4MemoryHelper.Peek32(ip)))
                {
                    token = op++;
                    *token = 0;
                    goto _next_match;
                }

                forwardH = HashPosition(++ip, tableType);
            }

_last_literals:
            {
                int lastRun = (int)(iend - anchor);
                if ((outputLimited == LimitedOutputDirective.LimitedOutput) &&
                    (op - dest + lastRun + 1 + (lastRun + 255 - RunMask) / 255 > (uint)maxOutputSize))
                {
                    return(0);
                }

                if (lastRun >= RunMask)
                {
                    int accumulator = (int)(lastRun - RunMask);

                    *op++ = (byte)(RunMask << MLBits);
                    for (; accumulator >= 255; accumulator -= 255)
                    {
                        *op++ = 255;
                    }
                    *op++ = (byte)accumulator;
                }
                else
                {
                    *op++ = (byte)(lastRun << MLBits);
                }

                LZ4MemoryHelper.Copy(op, anchor, lastRun);
                op += lastRun;
            }

            return((int)(op - dest));
        }
Example #4
0
        private static int EncodeSequence(byte **ip, byte **op, byte **anchor, int matchLength, byte *match, LimitedOutputDirective limit, byte *oend)
        {
            byte *token = (*op)++;

            size_t length = (size_t)(*ip - *anchor);

            if ((limit != LimitedOutputDirective.NoLimit) &&
                (*op + (length >> 8) + length + (2 + 1 + LastLiterals) > oend))
            {
                return(1);
            }

            if (length >= RunMask)
            {
                size_t len   = length - RunMask;
                *      token = (byte)(RunMask << MLBits);
                for (; len >= 255; len -= 255)
                {
                    *(*op)++ = 255;
                }
                *(*op)++ = (byte)len;
            }
            else
            {
                *token = (byte)(length << MLBits);
            }

            LZ4MemoryHelper.WildCopy(*op, *anchor, (*op) + length);

            *op += length;
            LZ4MemoryHelper.Poke16(*op, (ushort)(*ip - match));
            *op += 2;

            length = (size_t)(matchLength - MinMatch);
            if ((limit != LimitedOutputDirective.NoLimit) &&
                (*op + (length >> 8) + (1 + LastLiterals) > oend))
            {
                return(1);
            }

            if (length >= MLMask)
            {
                *token += (byte)MLMask;
                length -= MLMask;
                for (; length >= 510; length -= 510)
                {
                    *(*op)++ = 255;
                    *(*op)++ = 255;
                }

                if (length >= 255)
                {
                    length  -= 255;
                    *(*op)++ = 255;
                }

                *(*op)++ = (byte)length;
            }
            else
            {
                *token += (byte)(length);
            }

            *ip    += matchLength;
            *anchor = *ip;
            return(0);
        }