private static nuint ZDICT_addEntropyTablesFromBuffer_advanced(void *dictBuffer, nuint dictContentSize, nuint dictBufferCapacity, void *samplesBuffer, nuint *samplesSizes, uint nbSamples, ZDICT_params_t @params) { int compressionLevel = (@params.compressionLevel == 0) ? 3 : @params.compressionLevel; uint notificationLevel = @params.notificationLevel; nuint hSize = 8; { nuint eSize = ZDICT_analyzeEntropy((void *)((sbyte *)(dictBuffer) + hSize), dictBufferCapacity - hSize, compressionLevel, samplesBuffer, samplesSizes, nbSamples, (void *)((sbyte *)(dictBuffer) + dictBufferCapacity - dictContentSize), dictContentSize, notificationLevel); if ((ZDICT_isError(eSize)) != 0) { return(eSize); } hSize += eSize; } MEM_writeLE32(dictBuffer, 0xEC30A437); { ulong randomID = XXH64((void *)((sbyte *)(dictBuffer) + dictBufferCapacity - dictContentSize), dictContentSize, 0); uint compliantID = (uint)((randomID % ((1U << 31) - 32768)) + 32768); uint dictID = @params.dictID != 0 ? @params.dictID : compliantID; MEM_writeLE32((void *)((sbyte *)(dictBuffer) + 4), dictID); } if (hSize + dictContentSize < dictBufferCapacity) { memmove((void *)((sbyte *)(dictBuffer) + hSize), (void *)((sbyte *)(dictBuffer) + dictBufferCapacity - dictContentSize), dictContentSize); } return((dictBufferCapacity) < (hSize + dictContentSize) ? (dictBufferCapacity) : (hSize + dictContentSize)); }
/*! ZDICT_finalizeDictionary(): * Given a custom content as a basis for dictionary, and a set of samples, * finalize dictionary by adding headers and statistics according to the zstd * dictionary format. * * Samples must be stored concatenated in a flat buffer `samplesBuffer`, * supplied with an array of sizes `samplesSizes`, providing the size of each * sample in order. The samples are used to construct the statistics, so they * should be representative of what you will compress with this dictionary. * * The compression level can be set in `parameters`. You should pass the * compression level you expect to use in production. The statistics for each * compression level differ, so tuning the dictionary for the compression level * can help quite a bit. * * You can set an explicit dictionary ID in `parameters`, or allow us to pick * a random dictionary ID for you, but we can't guarantee no collisions. * * The dstDictBuffer and the dictContent may overlap, and the content will be * appended to the end of the header. If the header + the content doesn't fit in * maxDictSize the beginning of the content is truncated to make room, since it * is presumed that the most profitable content is at the end of the dictionary, * since that is the cheapest to reference. * * `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes. * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). * * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), * or an error code, which can be tested by ZDICT_isError(). * Note: ZDICT_finalizeDictionary() will push notifications into stderr if * instructed to, using notificationLevel>0. * NOTE: This function currently may fail in several edge cases including: * * Not enough samples * * Samples are uncompressible * * Samples are all exactly the same */ public static nuint ZDICT_finalizeDictionary(void *dictBuffer, nuint dictBufferCapacity, void *customDictContent, nuint dictContentSize, void *samplesBuffer, nuint *samplesSizes, uint nbSamples, ZDICT_params_t @params) { nuint hSize; byte *header = stackalloc byte[256]; int compressionLevel = (@params.compressionLevel == 0) ? 3 : @params.compressionLevel; uint notificationLevel = @params.notificationLevel; if (dictBufferCapacity < dictContentSize) { return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))); } if (dictContentSize < 128) { return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong))); } if (dictBufferCapacity < 256) { return(unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))); } MEM_writeLE32((void *)header, 0xEC30A437); { ulong randomID = XXH64(customDictContent, dictContentSize, 0); uint compliantID = (uint)((randomID % ((1U << 31) - 32768)) + 32768); uint dictID = @params.dictID != 0 ? @params.dictID : compliantID; MEM_writeLE32((void *)(header + 4), dictID); } hSize = 8; { nuint eSize = ZDICT_analyzeEntropy((void *)(header + hSize), 256 - hSize, compressionLevel, samplesBuffer, samplesSizes, nbSamples, customDictContent, dictContentSize, notificationLevel); if ((ZDICT_isError(eSize)) != 0) { return(eSize); } hSize += eSize; } if (hSize + dictContentSize > dictBufferCapacity) { dictContentSize = dictBufferCapacity - hSize; } { nuint dictSize = hSize + dictContentSize; sbyte *dictEnd = (sbyte *)(dictBuffer) + dictSize; memmove((void *)(dictEnd - dictContentSize), customDictContent, dictContentSize); memcpy(dictBuffer, (void *)header, hSize); return(dictSize); } }