private static nuint ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, byte *codeTable, uint maxCode, nuint nbSeq, uint *fseCTable, uint *additionalBits, short *defaultNorm, uint defaultNormLog, uint defaultMax, void *workspace, nuint wkspSize) { uint *countWksp = (uint *)(workspace); byte *ctp = codeTable; byte *ctStart = ctp; byte *ctEnd = ctStart + nbSeq; nuint cSymbolTypeSizeEstimateInBits = 0; uint max = maxCode; HIST_countFast_wksp(countWksp, &max, (void *)codeTable, nbSeq, workspace, wkspSize); if (type == symbolEncodingType_e.set_basic) { assert(max <= defaultMax); cSymbolTypeSizeEstimateInBits = max <= defaultMax?ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) : (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))); } else if (type == symbolEncodingType_e.set_rle) { cSymbolTypeSizeEstimateInBits = 0; } else if (type == symbolEncodingType_e.set_compressed || type == symbolEncodingType_e.set_repeat) { cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); } if ((ERR_isError(cSymbolTypeSizeEstimateInBits)) != 0) { return(nbSeq * 10); } while (ctp < ctEnd) { if (additionalBits != null) { cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; } else { cSymbolTypeSizeEstimateInBits += *ctp; } ctp++; } return(cSymbolTypeSizeEstimateInBits / 8); }
public static nuint ZSTD_compressLiterals(ZSTD_hufCTables_t *prevHuf, ZSTD_hufCTables_t *nextHuf, ZSTD_strategy strategy, int disableLiteralCompression, void *dst, nuint dstCapacity, void *src, nuint srcSize, void *entropyWorkspace, nuint entropyWorkspaceSize, int bmi2) { nuint minGain = ZSTD_minGain(srcSize, strategy); nuint lhSize = (nuint)(3 + ((srcSize >= (uint)(1 * (1 << 10))) ? 1 : 0) + ((srcSize >= (uint)(16 * (1 << 10))) ? 1 : 0)); byte *ostart = (byte *)(dst); uint singleStream = ((srcSize < 256) ? 1U : 0U); symbolEncodingType_e hType = symbolEncodingType_e.set_compressed; nuint cLitSize; memcpy((void *)(nextHuf), (void *)(prevHuf), ((nuint)(sizeof(ZSTD_hufCTables_t)))); if (disableLiteralCompression != 0) { return(ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize)); } { nuint minLitSize = (nuint)((prevHuf->repeatMode == HUF_repeat.HUF_repeat_valid) ? 6 : 63); if (srcSize <= minLitSize) { return(ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize)); } } if (dstCapacity < lhSize + 1) { return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))); } { HUF_repeat repeat = prevHuf->repeatMode; int preferRepeat = strategy < ZSTD_strategy.ZSTD_lazy ? ((srcSize <= 1024) ? 1 : 0) : 0; if (repeat == HUF_repeat.HUF_repeat_valid && lhSize == 3) { singleStream = 1; } cLitSize = singleStream != 0 ? HUF_compress1X_repeat((void *)(ostart + lhSize), dstCapacity - lhSize, src, srcSize, 255, 11, entropyWorkspace, entropyWorkspaceSize, (HUF_CElt_s *)(nextHuf->CTable), &repeat, preferRepeat, bmi2) : HUF_compress4X_repeat((void *)(ostart + lhSize), dstCapacity - lhSize, src, srcSize, 255, 11, entropyWorkspace, entropyWorkspaceSize, (HUF_CElt_s *)(nextHuf->CTable), &repeat, preferRepeat, bmi2); if (repeat != HUF_repeat.HUF_repeat_none) { hType = symbolEncodingType_e.set_repeat; } } if (((uint)(((((cLitSize == 0) || (cLitSize >= srcSize - minGain))) ? 1 : 0)) | ERR_isError(cLitSize)) != 0) { memcpy((void *)(nextHuf), (void *)(prevHuf), ((nuint)(sizeof(ZSTD_hufCTables_t)))); return(ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize)); } if (cLitSize == 1) { memcpy((void *)(nextHuf), (void *)(prevHuf), ((nuint)(sizeof(ZSTD_hufCTables_t)))); return(ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize)); } if (hType == symbolEncodingType_e.set_compressed) { nextHuf->repeatMode = HUF_repeat.HUF_repeat_check; } switch (lhSize) { case 3: { uint lhc = (uint)(hType + ((singleStream == 0 ? 1 : 0) << 2)) + ((uint)(srcSize) << 4) + ((uint)(cLitSize) << 14); MEM_writeLE24((void *)ostart, lhc); break; } case 4: { uint lhc = (uint)(hType + (2 << 2)) + ((uint)(srcSize) << 4) + ((uint)(cLitSize) << 18); MEM_writeLE32((void *)ostart, lhc); break; } case 5: { uint lhc = (uint)(hType + (3 << 2)) + ((uint)(srcSize) << 4) + ((uint)(cLitSize) << 22); MEM_writeLE32((void *)ostart, lhc); ostart[4] = (byte)(cLitSize >> 10); break; } default: { assert(0 != 0); } break; } return(lhSize + cLitSize); }
public static nuint ZSTD_buildCTable(void *dst, nuint dstCapacity, uint *nextCTable, uint FSELog, symbolEncodingType_e type, uint *count, uint max, byte *codeTable, nuint nbSeq, short *defaultNorm, uint defaultNormLog, uint defaultMax, uint *prevCTable, nuint prevCTableSize, void *entropyWorkspace, nuint entropyWorkspaceSize) { byte *op = (byte *)(dst); byte *oend = op + dstCapacity; switch (type) { case symbolEncodingType_e.set_rle: { { nuint err_code = (FSE_buildCTable_rle(nextCTable, (byte)(max))); if ((ERR_isError(err_code)) != 0) { return(err_code); } } } if (dstCapacity == 0) { return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))); } *op = codeTable[0]; return(1); case symbolEncodingType_e.set_repeat: { memcpy((void *)(nextCTable), (void *)(prevCTable), (prevCTableSize)); } return(0); case symbolEncodingType_e.set_basic: { { nuint err_code = (FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); if ((ERR_isError(err_code)) != 0) { return(err_code); } } } return(0); case symbolEncodingType_e.set_compressed: { short *norm = stackalloc short[53]; nuint nbSeq_1 = nbSeq; uint tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); if (count[codeTable[nbSeq - 1]] > 1) { count[codeTable[nbSeq - 1]]--; nbSeq_1--; } assert(nbSeq_1 > 1); assert(entropyWorkspaceSize >= ((nuint)(sizeof(uint)) * ((uint)(((35) > (52) ? (35) : (52)) + 2) + (1UL << (((((9) > (9) ? (9) : (9))) > (8) ? (((9) > (9) ? (9) : (9))) : (8)) - 2))))); { nuint err_code = (FSE_normalizeCount((short *)norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1))); if ((ERR_isError(err_code)) != 0) { return(err_code); } } { nuint NCountSize = FSE_writeNCount((void *)op, (nuint)(oend - op), (short *)norm, max, tableLog); { nuint err_code = (NCountSize); if ((ERR_isError(err_code)) != 0) { return(err_code); } } { nuint err_code = (FSE_buildCTable_wksp(nextCTable, (short *)norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize)); if ((ERR_isError(err_code)) != 0) { return(err_code); } } return(NCountSize); } } default: { assert(0 != 0); } { return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))); } } }
/** ZSTD_compressSubBlock_literal() : * Compresses literals section for a sub-block. * When we have to write the Huffman table we will sometimes choose a header * size larger than necessary. This is because we have to pick the header size * before we know the table size + compressed size, so we have a bound on the * table size. If we guessed incorrectly, we fall back to uncompressed literals. * * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded * in writing the header, otherwise it is set to 0. * * hufMetadata->hType has literals block type info. * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block * and the following sub-blocks' literals sections will be Treeless_Literals_Block. * @return : compressed size of literals section of a sub-block * Or 0 if it unable to compress. * Or error code */ private static nuint ZSTD_compressSubBlock_literal(HUF_CElt_s *hufTable, ZSTD_hufCTablesMetadata_t *hufMetadata, byte *literals, nuint litSize, void *dst, nuint dstSize, int bmi2, int writeEntropy, int *entropyWritten) { nuint header = (nuint)(writeEntropy != 0 ? 200 : 0); nuint lhSize = (nuint)(3 + ((litSize >= ((uint)(1 * (1 << 10)) - header)) ? 1 : 0) + ((litSize >= ((uint)(16 * (1 << 10)) - header)) ? 1 : 0)); byte *ostart = (byte *)(dst); byte *oend = ostart + dstSize; byte *op = ostart + lhSize; uint singleStream = ((lhSize == 3) ? 1U : 0U); symbolEncodingType_e hType = writeEntropy != 0 ? hufMetadata->hType : symbolEncodingType_e.set_repeat; nuint cLitSize = 0; *entropyWritten = 0; if (litSize == 0 || hufMetadata->hType == symbolEncodingType_e.set_basic) { return(ZSTD_noCompressLiterals(dst, dstSize, (void *)literals, litSize)); } else if (hufMetadata->hType == symbolEncodingType_e.set_rle) { return(ZSTD_compressRleLiteralsBlock(dst, dstSize, (void *)literals, litSize)); } assert(litSize > 0); assert(hufMetadata->hType == symbolEncodingType_e.set_compressed || hufMetadata->hType == symbolEncodingType_e.set_repeat); if (writeEntropy != 0 && hufMetadata->hType == symbolEncodingType_e.set_compressed) { memcpy((void *)(op), (void *)(hufMetadata->hufDesBuffer), (hufMetadata->hufDesSize)); op += hufMetadata->hufDesSize; cLitSize += hufMetadata->hufDesSize; } { nuint cSize = singleStream != 0 ? HUF_compress1X_usingCTable((void *)op, (nuint)(oend - op), (void *)literals, litSize, hufTable) : HUF_compress4X_usingCTable((void *)op, (nuint)(oend - op), (void *)literals, litSize, hufTable); op += cSize; cLitSize += cSize; if (cSize == 0 || (ERR_isError(cSize)) != 0) { return(0); } if (writeEntropy == 0 && cLitSize >= litSize) { return(ZSTD_noCompressLiterals(dst, dstSize, (void *)literals, litSize)); } if (lhSize < (nuint)(3 + ((cLitSize >= (uint)(1 * (1 << 10))) ? 1 : 0) + ((cLitSize >= (uint)(16 * (1 << 10))) ? 1 : 0))) { assert(cLitSize > litSize); return(ZSTD_noCompressLiterals(dst, dstSize, (void *)literals, litSize)); } } switch (lhSize) { case 3: { uint lhc = (uint)(hType + ((singleStream == 0 ? 1 : 0) << 2)) + ((uint)(litSize) << 4) + ((uint)(cLitSize) << 14); MEM_writeLE24((void *)ostart, lhc); break; } case 4: { uint lhc = (uint)(hType + (2 << 2)) + ((uint)(litSize) << 4) + ((uint)(cLitSize) << 18); MEM_writeLE32((void *)ostart, lhc); break; } case 5: { uint lhc = (uint)(hType + (3 << 2)) + ((uint)(litSize) << 4) + ((uint)(cLitSize) << 22); MEM_writeLE32((void *)ostart, lhc); ostart[4] = (byte)(cLitSize >> 10); break; } default: { assert(0 != 0); } break; } *entropyWritten = 1; return((nuint)(op - ostart)); }