private void ApplyBlockSplit(BlockSwitchBuilder builder, List <int> lengths)
{
builder.Reset();
if (lengths.Count < 2)
{
return;
}
for (int i = 0; i < lengths.Count; i++)
{
builder.AddBlock((byte)(i % 256), lengths[i]);
}
}
public void ProcessSequence(BlockSwitchBuilder builder, List <T> sequence, ContextInfo?contextParam = null)
{
builder.Reset();
ContextInfo ctx = contextParam ?? ContextInfo.Default;
int numContexts = 1 + ctx.Map.Max();
int maxBlockTypes = 256 / numContexts;
int maxNumBlocks = sequence.Count / minBlockSize + 1;
var histograms = FrequencyList <T> .Array(numContexts *Math.Min(maxNumBlocks, maxBlockTypes + 1));
int numBlocks = 0;
int blockSize = 0;
int targetBlockSize = minBlockSize;
int currHistogramIx = 0;
var lastHistogramIx = new int[2];
var lastEntropy = new double[2 * numContexts];
int mergeLastCount = 0;
void FinishBlock(bool isFinal)
{
blockSize = Math.Max(blockSize, minBlockSize);
if (numBlocks == 0)
{
builder.SetInitialLength(blockSize);
for (int context = 0; context < numContexts; context++)
{
lastEntropy[context] = BitsEntropy(histograms[context]);
lastEntropy[context + numContexts] = lastEntropy[context];
}
currHistogramIx += numContexts;
++numBlocks;
blockSize = 0;
}
else if (blockSize > 0)
{
var entropy = new double[numContexts];
var combinedHisto = FrequencyList <T> .Array(2 *numContexts);
var combinedEntropy = new double[2 * numContexts];
var diff = new double[2];
for (int context = 0; context < numContexts; context++)
{
int currHistoIx = currHistogramIx + context;
entropy[context] = BitsEntropy(histograms[currHistoIx]);
for (int i = 0; i < 2; i++)
{
int ix = i * numContexts + context;
int lastHistogramIxValue = lastHistogramIx[i] + context;
var newHisto = new FrequencyList <T> {
histograms[currHistoIx],
histograms[lastHistogramIxValue]
};
combinedHisto[ix] = newHisto;
combinedEntropy[ix] = BitsEntropy(combinedHisto[ix]);
diff[i] += combinedEntropy[ix] - entropy[context] - lastEntropy[ix];
}
}
if (builder.TypeCount < maxBlockTypes && diff[0] > splitThreshold && diff[1] > splitThreshold)
{
byte nextBlockType = (byte)(builder.TypeCount);
builder.AddBlock(nextBlockType, blockSize);
lastHistogramIx[1] = lastHistogramIx[0];
lastHistogramIx[0] = nextBlockType * numContexts;
for (int context = 0; context < numContexts; context++)
{
lastEntropy[context + numContexts] = lastEntropy[context];
lastEntropy[context] = entropy[context];
}
currHistogramIx += numContexts;
if (currHistogramIx < histograms.Length)
{
histograms[currHistogramIx].Clear();
}
++numBlocks;
blockSize = 0;
mergeLastCount = 0;
targetBlockSize = minBlockSize;
}
else if (diff[1] < diff[0] - 20.0)
{
builder.AddBlock(builder.Commands.Count >= 2 ? builder.Commands[^ 2].Type : (byte)0, blockSize);