private static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters * @params, uint *countLit, uint *offsetcodeCount, uint *matchlengthCount, uint *litlengthCount, uint *repOffsets, void *src, nuint srcSize, uint notificationLevel)
{
nuint blockSizeMax = (nuint)((((1 << 17)) < (1 << (int)@params->cParams.windowLog) ? ((1 << 17)) : (1 << (int)@params->cParams.windowLog)));
nuint cSize;
if (srcSize > blockSizeMax)
{
srcSize = blockSizeMax;
}
{
nuint errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict);
if ((ERR_isError(errorCode)) != 0)
{
return;
}
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, (nuint)((1 << 17)), src, srcSize);
if ((ERR_isError(cSize)) != 0)
{
return;
}
if (cSize != 0)
{
seqStore_t *seqStorePtr = ZSTD_getSeqStore(esr.zc);
{
byte *bytePtr;
for (bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
{
countLit[*bytePtr]++;
}
}
{
uint nbSeq = (uint)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
ZSTD_seqToCodes(seqStorePtr);
{
byte *codePtr = seqStorePtr->ofCode;
uint u;
for (u = 0; u < nbSeq; u++)
{
offsetcodeCount[codePtr[u]]++;
}
}
{
byte *codePtr = seqStorePtr->mlCode;
uint u;
for (u = 0; u < nbSeq; u++)
{
matchlengthCount[codePtr[u]]++;
}
}
{
byte *codePtr = seqStorePtr->llCode;
uint u;
for (u = 0; u < nbSeq; u++)
{
litlengthCount[codePtr[u]]++;
}
}
if (nbSeq >= 2)
{
seqDef_s *seq = seqStorePtr->sequencesStart;
uint offset1 = seq[0].offset - 3;
uint offset2 = seq[1].offset - 3;
if (offset1 >= 1024)
{
offset1 = 0;
}
if (offset2 >= 1024)
{
offset2 = 0;
}
repOffsets[offset1] += 3;
repOffsets[offset2] += 1;
}
}
}
}
private static nuint ZDICT_analyzeEntropy(void *dstBuffer, nuint maxDstSize, int compressionLevel, void *srcBuffer, nuint *fileSizes, uint nbFiles, void *dictBuffer, nuint dictBufferSize, uint notificationLevel)
{
uint * countLit = stackalloc uint[256];
HUF_CElt_s * hufTable = stackalloc HUF_CElt_s[256];
uint * offcodeCount = stackalloc uint[31];
short * offcodeNCount = stackalloc short[31];
uint offcodeMax = ZSTD_highbit32((uint)(dictBufferSize + (uint)(128 * (1 << 10))));
uint * matchLengthCount = stackalloc uint[53];
short * matchLengthNCount = stackalloc short[53];
uint * litLengthCount = stackalloc uint[36];
short * litLengthNCount = stackalloc short[36];
uint * repOffset = stackalloc uint[1024];
offsetCount_t *bestRepOffset = stackalloc offsetCount_t[4];
EStats_ress_t esr = new EStats_ress_t
{
dict = null,
zc = null,
workPlace = null,
};
ZSTD_parameters @params;
uint u, huffLog = 11, Offlog = 8, mlLog = 9, llLog = 9, total;
nuint pos = 0, errorCode;
nuint eSize = 0;
nuint totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
nuint averageSampleSize = totalSrcSize / (nbFiles + (uint)(nbFiles == 0 ? 1 : 0));
byte * dstPtr = (byte *)(dstBuffer);
if (offcodeMax > 30)
{
eSize = (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dictionaryCreation_failed)));
goto _cleanup;
}
for (u = 0; u < 256; u++)
{
countLit[u] = 1;
}
for (u = 0; u <= offcodeMax; u++)
{
offcodeCount[u] = 1;
}
for (u = 0; u <= 52; u++)
{
matchLengthCount[u] = 1;
}
for (u = 0; u <= 35; u++)
{
litLengthCount[u] = 1;
}
memset((void *)repOffset, 0, (nuint)(sizeof(uint) * 1024));
repOffset[1] = repOffset[4] = repOffset[8] = 1;
memset((void *)bestRepOffset, 0, (nuint)(sizeof(offsetCount_t) * 4));
if (compressionLevel == 0)
{
compressionLevel = 3;
}
@params = ZSTD_getParams(compressionLevel, (ulong)averageSampleSize, dictBufferSize);
esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dictLoadMethod_e.ZSTD_dlm_byRef, ZSTD_dictContentType_e.ZSTD_dct_rawContent, @params.cParams, ZSTD_defaultCMem);
esr.zc = ZSTD_createCCtx();
esr.workPlace = malloc((nuint)((1 << 17)));
if (esr.dict == null || esr.zc == null || esr.workPlace == null)
{
eSize = (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation)));
goto _cleanup;
}
for (u = 0; u < nbFiles; u++)
{
ZDICT_countEStats(esr, &@params, (uint *)countLit, (uint *)offcodeCount, (uint *)matchLengthCount, (uint *)litLengthCount, (uint *)repOffset, (void *)((sbyte *)(srcBuffer) + pos), fileSizes[u], notificationLevel);
pos += fileSizes[u];
}
{
nuint maxNbBits = HUF_buildCTable((HUF_CElt_s *)hufTable, (uint *)countLit, 255, huffLog);
if ((ERR_isError(maxNbBits)) != 0)
{
eSize = maxNbBits;
goto _cleanup;
}
if (maxNbBits == 8)
{
ZDICT_flatLit((uint *)countLit);
maxNbBits = HUF_buildCTable((HUF_CElt_s *)hufTable, (uint *)countLit, 255, huffLog);
assert(maxNbBits == 9);
}
huffLog = (uint)(maxNbBits);
}
{
uint offset;
for (offset = 1; offset < 1024; offset++)
{
ZDICT_insertSortCount(bestRepOffset, offset, repOffset[offset]);
}
}
total = 0;
for (u = 0; u <= offcodeMax; u++)
{
total += offcodeCount[u];
}
errorCode = FSE_normalizeCount((short *)offcodeNCount, Offlog, (uint *)offcodeCount, total, offcodeMax, 1);
if ((ERR_isError(errorCode)) != 0)
{
eSize = errorCode;
goto _cleanup;
}
Offlog = (uint)(errorCode);
total = 0;
for (u = 0; u <= 52; u++)
{
total += matchLengthCount[u];
}
errorCode = FSE_normalizeCount((short *)matchLengthNCount, mlLog, (uint *)matchLengthCount, total, 52, 1);
if ((ERR_isError(errorCode)) != 0)
{
eSize = errorCode;
goto _cleanup;
}
mlLog = (uint)(errorCode);
total = 0;
for (u = 0; u <= 35; u++)
{
total += litLengthCount[u];
}
errorCode = FSE_normalizeCount((short *)litLengthNCount, llLog, (uint *)litLengthCount, total, 35, 1);
if ((ERR_isError(errorCode)) != 0)
{
eSize = errorCode;
goto _cleanup;
}
llLog = (uint)(errorCode);
{
nuint hhSize = HUF_writeCTable((void *)dstPtr, maxDstSize, (HUF_CElt_s *)hufTable, 255, huffLog);
if ((ERR_isError(hhSize)) != 0)
{
eSize = hhSize;
goto _cleanup;
}
dstPtr += hhSize;
maxDstSize -= hhSize;
eSize += hhSize;
}
{
nuint ohSize = FSE_writeNCount((void *)dstPtr, maxDstSize, (short *)offcodeNCount, 30, Offlog);
if ((ERR_isError(ohSize)) != 0)
{
eSize = ohSize;
goto _cleanup;
}
dstPtr += ohSize;
maxDstSize -= ohSize;
eSize += ohSize;
}
{
nuint mhSize = FSE_writeNCount((void *)dstPtr, maxDstSize, (short *)matchLengthNCount, 52, mlLog);
if ((ERR_isError(mhSize)) != 0)
{
eSize = mhSize;
goto _cleanup;
}
dstPtr += mhSize;
maxDstSize -= mhSize;
eSize += mhSize;
}
{
nuint lhSize = FSE_writeNCount((void *)dstPtr, maxDstSize, (short *)litLengthNCount, 35, llLog);
if ((ERR_isError(lhSize)) != 0)
{
eSize = lhSize;
goto _cleanup;
}
dstPtr += lhSize;
maxDstSize -= lhSize;
eSize += lhSize;
}
if (maxDstSize < 12)
{
eSize = (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
goto _cleanup;
}
MEM_writeLE32((void *)(dstPtr + 0), repStartValue[0]);
MEM_writeLE32((void *)(dstPtr + 4), repStartValue[1]);
MEM_writeLE32((void *)(dstPtr + 8), repStartValue[2]);
eSize += 12;
_cleanup:
ZSTD_freeCDict(esr.dict);
ZSTD_freeCCtx(esr.zc);
free(esr.workPlace);
return(eSize);
}