private static nuint ZDICT_addEntropyTablesFromBuffer_advanced(void *dictBuffer, nuint dictContentSize, nuint dictBufferCapacity, void *samplesBuffer, nuint *samplesSizes, uint nbSamples, ZDICT_params_t @params)
{
int compressionLevel = (@params.compressionLevel == 0) ? 3 : @params.compressionLevel;
uint notificationLevel = @params.notificationLevel;
nuint hSize = 8;
{
nuint eSize = ZDICT_analyzeEntropy((void *)((sbyte *)(dictBuffer) + hSize), dictBufferCapacity - hSize, compressionLevel, samplesBuffer, samplesSizes, nbSamples, (void *)((sbyte *)(dictBuffer) + dictBufferCapacity - dictContentSize), dictContentSize, notificationLevel);
if ((ZDICT_isError(eSize)) != 0)
{
return(eSize);
}
hSize += eSize;
}
MEM_writeLE32(dictBuffer, 0xEC30A437);
{
ulong randomID = XXH64((void *)((sbyte *)(dictBuffer) + dictBufferCapacity - dictContentSize), dictContentSize, 0);
uint compliantID = (uint)((randomID % ((1U << 31) - 32768)) + 32768);
uint dictID = @params.dictID != 0 ? @params.dictID : compliantID;
MEM_writeLE32((void *)((sbyte *)(dictBuffer) + 4), dictID);
}
if (hSize + dictContentSize < dictBufferCapacity)
{
memmove((void *)((sbyte *)(dictBuffer) + hSize), (void *)((sbyte *)(dictBuffer) + dictBufferCapacity - dictContentSize), dictContentSize);
}
return((dictBufferCapacity) < (hSize + dictContentSize) ? (dictBufferCapacity) : (hSize + dictContentSize));
}
/*! ZDICT_finalizeDictionary():
* Given a custom content as a basis for dictionary, and a set of samples,
* finalize dictionary by adding headers and statistics according to the zstd
* dictionary format.
*
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
* supplied with an array of sizes `samplesSizes`, providing the size of each
* sample in order. The samples are used to construct the statistics, so they
* should be representative of what you will compress with this dictionary.
*
* The compression level can be set in `parameters`. You should pass the
* compression level you expect to use in production. The statistics for each
* compression level differ, so tuning the dictionary for the compression level
* can help quite a bit.
*
* You can set an explicit dictionary ID in `parameters`, or allow us to pick
* a random dictionary ID for you, but we can't guarantee no collisions.
*
* The dstDictBuffer and the dictContent may overlap, and the content will be
* appended to the end of the header. If the header + the content doesn't fit in
* maxDictSize the beginning of the content is truncated to make room, since it
* is presumed that the most profitable content is at the end of the dictionary,
* since that is the cheapest to reference.
*
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
*
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
* or an error code, which can be tested by ZDICT_isError().
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if
* instructed to, using notificationLevel>0.
* NOTE: This function currently may fail in several edge cases including:
* * Not enough samples
* * Samples are uncompressible
* * Samples are all exactly the same
*/
public static nuint ZDICT_finalizeDictionary(void *dictBuffer, nuint dictBufferCapacity, void *customDictContent, nuint dictContentSize, void *samplesBuffer, nuint *samplesSizes, uint nbSamples, ZDICT_params_t @params)
{
nuint hSize;
byte *header = stackalloc byte[256];
int compressionLevel = (@params.compressionLevel == 0) ? 3 : @params.compressionLevel;
uint notificationLevel = @params.notificationLevel;
if (dictBufferCapacity < dictContentSize)
{
return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
if (dictContentSize < 128)
{
return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (dictBufferCapacity < 256)
{
return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
MEM_writeLE32((void *)header, 0xEC30A437);
{
ulong randomID = XXH64(customDictContent, dictContentSize, 0);
uint compliantID = (uint)((randomID % ((1U << 31) - 32768)) + 32768);
uint dictID = @params.dictID != 0 ? @params.dictID : compliantID;
MEM_writeLE32((void *)(header + 4), dictID);
}
hSize = 8;
{
nuint eSize = ZDICT_analyzeEntropy((void *)(header + hSize), 256 - hSize, compressionLevel, samplesBuffer, samplesSizes, nbSamples, customDictContent, dictContentSize, notificationLevel);
if ((ZDICT_isError(eSize)) != 0)
{
return(eSize);
}
hSize += eSize;
}
if (hSize + dictContentSize > dictBufferCapacity)
{
dictContentSize = dictBufferCapacity - hSize;
}
{
nuint dictSize = hSize + dictContentSize;
sbyte *dictEnd = (sbyte *)(dictBuffer) + dictSize;
memmove((void *)(dictEnd - dictContentSize), customDictContent, dictContentSize);
memcpy(dictBuffer, (void *)header, hSize);
return(dictSize);
}
}
