/** * Error function for COVER_selectDict function. Returns a struct where * return.totalCompressedSize is a ZSTD error. */ public static COVER_dictSelection COVER_dictSelectionError(nuint error) { COVER_dictSelection selection = new COVER_dictSelection { dictContent = null, dictSize = 0, totalCompressedSize = error, }; return(selection); }
/** * Tries a set of parameters and updates the COVER_best_t with the results. * This function is thread safe if zstd is compiled with multithreaded support. * It takes its parameters as an *OWNING* opaque pointer to support threading. */ private static void FASTCOVER_tryParameters(void *opaque) { FASTCOVER_tryParameters_data_s *data = (FASTCOVER_tryParameters_data_s *)(opaque); FASTCOVER_ctx_t * ctx = data->ctx; ZDICT_cover_params_t parameters = data->parameters; nuint dictBufferCapacity = data->dictBufferCapacity; nuint totalCompressedSize = (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))); ushort * segmentFreqs = (ushort *)(calloc((nuint)((ulong)(1) << (int)ctx->f), (nuint)(2))); byte * dict = (byte *)(malloc(dictBufferCapacity)); COVER_dictSelection selection = COVER_dictSelectionError((unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)))); uint * freqs = (uint *)(malloc((nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(4)))); if (segmentFreqs == null || dict == null || freqs == null) { goto _cleanup; } memcpy((void *)freqs, (void *)ctx->freqs, (nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(sizeof(uint)))); { nuint tail = FASTCOVER_buildDictionary(ctx, freqs, (void *)dict, dictBufferCapacity, parameters, segmentFreqs); uint nbFinalizeSamples = (uint)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100); selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, totalCompressedSize); if ((COVER_dictSelectionIsError(selection)) != 0) { goto _cleanup; } } _cleanup: free((void *)dict); COVER_best_finish(data->best, parameters, selection); free((void *)data); free((void *)segmentFreqs); COVER_dictSelectionFree(selection); free((void *)freqs); }
/** * Called to finalize the dictionary and select one based on whether or not * the shrink-dict flag was enabled. If enabled the dictionary used is the * smallest dictionary within a specified regression of the compressed size * from the largest dictionary. */ public static COVER_dictSelection COVER_selectDict(byte *customDictContent, nuint dictBufferCapacity, nuint dictContentSize, byte *samplesBuffer, nuint *samplesSizes, uint nbFinalizeSamples, nuint nbCheckSamples, nuint nbSamples, ZDICT_cover_params_t @params, nuint *offsets, nuint totalCompressedSize) { nuint largestDict = 0; nuint largestCompressed = 0; byte * customDictContentEnd = customDictContent + dictContentSize; byte * largestDictbuffer = (byte *)(malloc(dictBufferCapacity)); byte * candidateDictBuffer = (byte *)(malloc(dictBufferCapacity)); double regressionTolerance = ((double)(@params.shrinkDictMaxRegression) / 100.0) + 1.00; if (largestDictbuffer == null || candidateDictBuffer == null) { free((void *)largestDictbuffer); free((void *)candidateDictBuffer); return(COVER_dictSelectionError(dictContentSize)); } memcpy((void *)largestDictbuffer, (void *)customDictContent, dictContentSize); dictContentSize = ZDICT_finalizeDictionary((void *)largestDictbuffer, dictBufferCapacity, (void *)customDictContent, dictContentSize, (void *)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams); if ((ZDICT_isError(dictContentSize)) != 0) { free((void *)largestDictbuffer); free((void *)candidateDictBuffer); return(COVER_dictSelectionError(dictContentSize)); } totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, largestDictbuffer, dictContentSize); if ((ERR_isError(totalCompressedSize)) != 0) { free((void *)largestDictbuffer); free((void *)candidateDictBuffer); return(COVER_dictSelectionError(totalCompressedSize)); } if (@params.shrinkDict == 0) { COVER_dictSelection selection = new COVER_dictSelection { dictContent = largestDictbuffer, dictSize = dictContentSize, totalCompressedSize = totalCompressedSize, }; free((void *)candidateDictBuffer); return(selection); } largestDict = dictContentSize; largestCompressed = totalCompressedSize; dictContentSize = 256; while (dictContentSize < largestDict) { memcpy((void *)candidateDictBuffer, (void *)largestDictbuffer, largestDict); dictContentSize = ZDICT_finalizeDictionary((void *)candidateDictBuffer, dictBufferCapacity, (void *)(customDictContentEnd - dictContentSize), dictContentSize, (void *)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams); if ((ZDICT_isError(dictContentSize)) != 0) { free((void *)largestDictbuffer); free((void *)candidateDictBuffer); return(COVER_dictSelectionError(dictContentSize)); } totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, candidateDictBuffer, dictContentSize); if ((ERR_isError(totalCompressedSize)) != 0) { free((void *)largestDictbuffer); free((void *)candidateDictBuffer); return(COVER_dictSelectionError(totalCompressedSize)); } if (totalCompressedSize <= largestCompressed * regressionTolerance) { COVER_dictSelection selection = new COVER_dictSelection { dictContent = candidateDictBuffer, dictSize = dictContentSize, totalCompressedSize = totalCompressedSize, }; free((void *)largestDictbuffer); return(selection); } dictContentSize *= 2; } dictContentSize = largestDict; totalCompressedSize = largestCompressed; { COVER_dictSelection selection = new COVER_dictSelection { dictContent = largestDictbuffer, dictSize = dictContentSize, totalCompressedSize = totalCompressedSize, }; free((void *)candidateDictBuffer); return(selection); } }
/** * Always call after selectDict is called to free up used memory from * newly created dictionary. */ public static void COVER_dictSelectionFree(COVER_dictSelection selection) { free((void *)selection.dictContent); }
/** * Error function for COVER_selectDict function. Checks if the return * value is an error. */ public static uint COVER_dictSelectionIsError(COVER_dictSelection selection) { return((((ERR_isError(selection.totalCompressedSize)) != 0 || selection.dictContent == null)) ? 1U : 0U); }
/** * Called when a thread finishes executing, both on error or success. * Decrements liveJobs and signals any waiting threads if liveJobs == 0. * If this dictionary is the best so far save it and its parameters. */ public static void COVER_best_finish(COVER_best_s *best, ZDICT_cover_params_t parameters, COVER_dictSelection selection) { void *dict = (void *)selection.dictContent; nuint compressedSize = selection.totalCompressedSize; nuint dictSize = selection.dictSize; if (best == null) { return; } { nuint liveJobs; --best->liveJobs; liveJobs = best->liveJobs; if (compressedSize < best->compressedSize) { if (best->dict == null || best->dictSize < dictSize) { if (best->dict != null) { free(best->dict); } best->dict = malloc(dictSize); if (best->dict == null) { best->compressedSize = (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))); best->dictSize = 0; return; } } if (dict != null) { memcpy(best->dict, dict, dictSize); best->dictSize = dictSize; best->parameters = parameters; best->compressedSize = compressedSize; } } if (liveJobs == 0) { } } }