/**
* Error function for COVER_selectDict function. Returns a struct where
* return.totalCompressedSize is a ZSTD error.
*/
public static COVER_dictSelection COVER_dictSelectionError(nuint error)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = null,
dictSize = 0,
totalCompressedSize = error,
};
return(selection);
}
/**
* Tries a set of parameters and updates the COVER_best_t with the results.
* This function is thread safe if zstd is compiled with multithreaded support.
* It takes its parameters as an *OWNING* opaque pointer to support threading.
*/
private static void FASTCOVER_tryParameters(void *opaque)
{
FASTCOVER_tryParameters_data_s *data = (FASTCOVER_tryParameters_data_s *)(opaque);
FASTCOVER_ctx_t * ctx = data->ctx;
ZDICT_cover_params_t parameters = data->parameters;
nuint dictBufferCapacity = data->dictBufferCapacity;
nuint totalCompressedSize = (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
ushort * segmentFreqs = (ushort *)(calloc((nuint)((ulong)(1) << (int)ctx->f), (nuint)(2)));
byte * dict = (byte *)(malloc(dictBufferCapacity));
COVER_dictSelection selection = COVER_dictSelectionError((unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))));
uint * freqs = (uint *)(malloc((nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(4))));
if (segmentFreqs == null || dict == null || freqs == null)
{
goto _cleanup;
}
memcpy((void *)freqs, (void *)ctx->freqs, (nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(sizeof(uint))));
{
nuint tail = FASTCOVER_buildDictionary(ctx, freqs, (void *)dict, dictBufferCapacity, parameters, segmentFreqs);
uint nbFinalizeSamples = (uint)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, totalCompressedSize);
if ((COVER_dictSelectionIsError(selection)) != 0)
{
goto _cleanup;
}
}
_cleanup:
free((void *)dict);
COVER_best_finish(data->best, parameters, selection);
free((void *)data);
free((void *)segmentFreqs);
COVER_dictSelectionFree(selection);
free((void *)freqs);
}
/**
* Called to finalize the dictionary and select one based on whether or not
* the shrink-dict flag was enabled. If enabled the dictionary used is the
* smallest dictionary within a specified regression of the compressed size
* from the largest dictionary.
*/
public static COVER_dictSelection COVER_selectDict(byte *customDictContent, nuint dictBufferCapacity, nuint dictContentSize, byte *samplesBuffer, nuint *samplesSizes, uint nbFinalizeSamples, nuint nbCheckSamples, nuint nbSamples, ZDICT_cover_params_t @params, nuint *offsets, nuint totalCompressedSize)
{
nuint largestDict = 0;
nuint largestCompressed = 0;
byte * customDictContentEnd = customDictContent + dictContentSize;
byte * largestDictbuffer = (byte *)(malloc(dictBufferCapacity));
byte * candidateDictBuffer = (byte *)(malloc(dictBufferCapacity));
double regressionTolerance = ((double)(@params.shrinkDictMaxRegression) / 100.0) + 1.00;
if (largestDictbuffer == null || candidateDictBuffer == null)
{
free((void *)largestDictbuffer);
free((void *)candidateDictBuffer);
return(COVER_dictSelectionError(dictContentSize));
}
memcpy((void *)largestDictbuffer, (void *)customDictContent, dictContentSize);
dictContentSize = ZDICT_finalizeDictionary((void *)largestDictbuffer, dictBufferCapacity, (void *)customDictContent, dictContentSize, (void *)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams);
if ((ZDICT_isError(dictContentSize)) != 0)
{
free((void *)largestDictbuffer);
free((void *)candidateDictBuffer);
return(COVER_dictSelectionError(dictContentSize));
}
totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, largestDictbuffer, dictContentSize);
if ((ERR_isError(totalCompressedSize)) != 0)
{
free((void *)largestDictbuffer);
free((void *)candidateDictBuffer);
return(COVER_dictSelectionError(totalCompressedSize));
}
if (@params.shrinkDict == 0)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = largestDictbuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void *)candidateDictBuffer);
return(selection);
}
largestDict = dictContentSize;
largestCompressed = totalCompressedSize;
dictContentSize = 256;
while (dictContentSize < largestDict)
{
memcpy((void *)candidateDictBuffer, (void *)largestDictbuffer, largestDict);
dictContentSize = ZDICT_finalizeDictionary((void *)candidateDictBuffer, dictBufferCapacity, (void *)(customDictContentEnd - dictContentSize), dictContentSize, (void *)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams);
if ((ZDICT_isError(dictContentSize)) != 0)
{
free((void *)largestDictbuffer);
free((void *)candidateDictBuffer);
return(COVER_dictSelectionError(dictContentSize));
}
totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, candidateDictBuffer, dictContentSize);
if ((ERR_isError(totalCompressedSize)) != 0)
{
free((void *)largestDictbuffer);
free((void *)candidateDictBuffer);
return(COVER_dictSelectionError(totalCompressedSize));
}
if (totalCompressedSize <= largestCompressed * regressionTolerance)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = candidateDictBuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void *)largestDictbuffer);
return(selection);
}
dictContentSize *= 2;
}
dictContentSize = largestDict;
totalCompressedSize = largestCompressed;
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = largestDictbuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void *)candidateDictBuffer);
return(selection);
}
}
/**
* Always call after selectDict is called to free up used memory from
* newly created dictionary.
*/
public static void COVER_dictSelectionFree(COVER_dictSelection selection)
{
free((void *)selection.dictContent);
}
/**
* Error function for COVER_selectDict function. Checks if the return
* value is an error.
*/
public static uint COVER_dictSelectionIsError(COVER_dictSelection selection)
{
return((((ERR_isError(selection.totalCompressedSize)) != 0 || selection.dictContent == null)) ? 1U : 0U);
}
/**
* Called when a thread finishes executing, both on error or success.
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
* If this dictionary is the best so far save it and its parameters.
*/
public static void COVER_best_finish(COVER_best_s *best, ZDICT_cover_params_t parameters, COVER_dictSelection selection)
{
void *dict = (void *)selection.dictContent;
nuint compressedSize = selection.totalCompressedSize;
nuint dictSize = selection.dictSize;
if (best == null)
{
return;
}
{
nuint liveJobs;
--best->liveJobs;
liveJobs = best->liveJobs;
if (compressedSize < best->compressedSize)
{
if (best->dict == null || best->dictSize < dictSize)
{
if (best->dict != null)
{
free(best->dict);
}
best->dict = malloc(dictSize);
if (best->dict == null)
{
best->compressedSize = (unchecked ((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
best->dictSize = 0;
return;
}
}
if (dict != null)
{
memcpy(best->dict, dict, dictSize);
best->dictSize = dictSize;
best->parameters = parameters;
best->compressedSize = compressedSize;
}
}
if (liveJobs == 0)
{
}
}
}
