protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state)
{
LiteralCtxMap = ContextMapBuilder.Literals.Simple,
DistanceCtxMap = ContextMapBuilder.Distances.Simple
};
foreach (var category in Categories.LID)
{
builder.BlockTypes[category].Reset();
}
builder.UseSameLiteralContextMode(LiteralContextMode.UTF8);
var tracker = new MetaBlockSizeTracker(state);
tracker.Test(builder, parameters);
foreach (var category in Categories.LID)
{
TestBlockSplits(builder, parameters, tracker, category);
}
tracker.Test(original);
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
private NextStep EmitRemainder(CompressedMetaBlockBuilder builder)
{
nextIpStart += blockSize;
inputSize -= blockSize;
blockSize = Math.Min(inputSize, MergeBlockSize);
if (inputSize > 0 && totalBlockSize + blockSize <= (1 << 20) && ShouldMergeBlock(nextIpStart, blockSize))
{
totalBlockSize += blockSize;
return(NextStep.EmitCommands);
}
if (nextEmit < ipEnd)
{
int insert = ipEnd - nextEmit;
if (insert >= 6210 && ShouldUseUncompressedMode(insert))
{
outputTo = ipEnd;
return(NextStep.OutputUncompressed);
}
else
{
builder.AddInsert(Literal.FromBytes(input, nextEmit, insert));
}
}
nextEmit = ipEnd;
return(NextStep.OutputCompressed);
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state)
{
LiteralCtxMap = ContextMapBuilder.Literals.Simple,
DistanceCtxMap = ContextMapBuilder.Distances.Simple
};
foreach (var category in Categories.LID)
{
builder.BlockTypes[category].Reset();
}
return(builder.UseSameLiteralContextMode(LiteralContextMode.LSB6).Build(parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
if (original.Data.InsertCopyCommands.Count <= 1 || original.Data.BlockSwitchCommands[Category.InsertCopy].Count > 0)
{
return(original, state);
}
var builder = new CompressedMetaBlockBuilder(original, state);
var icBlocks = builder.BlockTypes[Category.InsertCopy];
var icCommands = builder.GetTotalBlockLength(Category.InsertCopy);
icBlocks.SetInitialLength(icCommands / 2)
.AddFinalBlock(1);
return(builder.Build(parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var tracker = new MetaBlockSizeTracker(state);
for (byte postfixBitCount = 0; postfixBitCount <= DistanceParameters.MaxPostfixBitCount; postfixBitCount++)
{
for (byte directCodeBits = 0; directCodeBits <= DistanceParameters.MaxDirectCodeBits; directCodeBits++)
{
builder.DistanceParameters = new DistanceParameters(postfixBitCount, directCodeBits);
tracker.Test(builder, parameters, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]");
}
}
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
private void PrepareContextMap(CompressedMetaBlockBuilder builder, Category category, int lengthSplits)
{
int blockTypes = Math.Min(lengthSplits, 256);
if (category == Category.Literal)
{
if (builder.LiteralCtxMap.BlockTypes != blockTypes)
{
builder.UseSameLiteralContextMode(LiteralContextMode.UTF8);
builder.LiteralCtxMap = new ContextMapBuilder.Literals(blockTypes).RepeatFirstBlockType(lengthSplits <= blockTypes).Build();
}
}
else if (category == Category.Distance)
{
if (builder.DistanceCtxMap.BlockTypes != blockTypes)
{
builder.DistanceCtxMap = new ContextMapBuilder.Distances(blockTypes).RepeatFirstBlockType(lengthSplits <= blockTypes).Build();
}
}
}
/// <summary>
/// Adapted from https://github.com/google/brotli/blob/master/c/enc/encode.c (EncodeData).
/// </summary>
private bool ShouldContinueThisBlock(CompressedMetaBlockBuilder builder)
{
int maxLength = MaxMetaBlockSize(fileParameters, lgBlock);
if (builder.OutputSize + InputBlockSize(lgBlock) > maxLength)
{
return(false);
}
int totalCommands = builder.InsertCopyCommands.Count;
int totalLiterals = builder.InsertCopyCommands.Sum(icCommand => icCommand.Literals.Count);
if (!features.HasFlag(Features.BlockSplit) && totalCommands + totalLiterals >= 0x2FFF /* 12287 */)
{
return(false);
}
int maxCommands = maxLength / 8;
int maxLiterals = maxLength / 8;
return(totalCommands < maxCommands && totalLiterals < maxLiterals);
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var tracker = new MetaBlockSizeTracker(state);
tracker.Test(builder, parameters);
var blocker = new Blocker(parameters.DistanceCodePicker);
parameters = new BrotliCompressionParameters.Builder(parameters)
{
DistanceCodePicker = blocker.Pick
}.Build();
foreach (var code in DistanceCode.Last.Codes.Except(new DistanceCode[] { DistanceCode.Zero }))
{
var prev = tracker.SmallestSize;
blocker.BlockedCodes.Add(code);
tracker.Test(builder, parameters);
if (tracker.SmallestSize < prev)
{
Debug.WriteLine("Blocking code " + code + " reduced size (" + prev + " > " + tracker.SmallestSize + "), keeping it...");
}
else
{
Debug.WriteLine("Blocking code " + code + " did not improve the size, continuing...");
blocker.BlockedCodes.Remove(code);
}
}
Debug.WriteLine("Final blocked codes: " + string.Join(", ", blocker.BlockedCodes));
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
private void Split <T>(CompressedMetaBlockBuilder builder, Category category, List <T> sequence, int minBlockSize, double splitThreshold, in BlockSplitter <T> .ContextInfo?context = null) where T : IComparable <T>
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var literals = new List <Literal>(builder.GetTotalBlockLength(Category.Literal));
var lengthCodes = new List <InsertCopyLengthCode>(builder.GetTotalBlockLength(Category.InsertCopy));
var distanceCodes = new List <DistanceCode>(builder.GetTotalBlockLength(Category.Distance));
var distanceFreq = new FrequencyList <DistanceCode>();
var validDistanceCodes = new List <DistanceCode>(5);
foreach (var command in original.Data.InsertCopyCommands)
{
literals.AddRange(command.Literals);
state.OutputLiterals(command.Literals);
if (command.CopyDistance == DistanceInfo.EndsAfterLiterals)
{
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.PreferEnabled));
break;
}
if (!command.CopyDistance.FindCodes(original.Header.DistanceParameters, state, validDistanceCodes))
{
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.ForceEnabled));
}
else
{
DistanceCode distanceCode;
if (command.CopyDistance == DistanceInfo.ExplicitCodeZero)
{
distanceCode = DistanceCode.Zero;
}
else
{
distanceCode = validDistanceCodes.Count > 1 ? parameters.DistanceCodePicker(validDistanceCodes, distanceFreq) : validDistanceCodes[0];
}
distanceFreq.Add(distanceCode);
distanceCodes.Add(distanceCode);
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.Disable));
}
}
var origLitCtxMap = builder.LiteralCtxMap;
if (origLitCtxMap.TreeCount == 1)
{
Split(builder, Category.Literal, literals, 512, 400.0);
builder.UseSameLiteralContextMode(LiteralContextMode.UTF8);
builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).RepeatFirstBlockType(true).Build();
}
else
{
var literalContextMap = Enumerable.Range(0, origLitCtxMap.ContextsPerBlockType).Select(index => origLitCtxMap.DetermineTreeID(0, index)).ToArray();
var literalContextMode = builder.LiteralContextModes[0];
var literalBuffer = RingBufferFast <byte> .From(0, 0);
Split(builder, Category.Literal, literals, 512, 400.0, new BlockSplitter <Literal> .ContextInfo(literalContextMap, literal => {
literalBuffer.Push(literal.Value);
return(literalContextMode.DetermineContextID(literalBuffer.Front, literalBuffer.Back));
}));
builder.UseSameLiteralContextMode(literalContextMode);
builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).Set(0, literalContextMap).RepeatFirstBlockType(true).Build();
}
Split(builder, Category.InsertCopy, lengthCodes, 1024, 500.0);
Split(builder, Category.Distance, distanceCodes, 512, 100.0);
builder.DistanceCtxMap = new ContextMapBuilder.Distances(builder).RepeatFirstBlockType(true).Build();
return(builder.Build(parameters));
}
/// <summary>
/// Adapted from https://github.com/google/brotli/blob/master/c/enc/backward_references.c (BrotliCreateBackwardReferences).
/// </summary>
private void CreateBackwardReferences(CompressedMetaBlockBuilder builder, int chunkLength)
{
int maxBackwardLimit = fileParameters.WindowSize.Bytes;
int insertLength = lastInsertLen;
int posEnd = position + chunkLength;
int storeEnd = chunkLength >= hasher.StoreLookahead ? posEnd - hasher.StoreLookahead + 1 : position;
int applyRandomHeuristics = position + LiteralSpreeLengthForSparseSearch;
while (position + hasher.HashTypeLength < posEnd)
{
int maxLength = posEnd - position;
int maxDistance = Math.Min(position, maxBackwardLimit);
int dictionaryStart = Math.Min(position, maxBackwardLimit);
var result = hasher.FindLongestMatch(position, maxLength, maxDistance, dictionaryStart, builder.LastDistance, 0);
if (result.FoundAnything)
{
int delayedBackwardReferencesInRow = 0;
--maxLength;
while (true)
{
maxDistance = Math.Min(position + 1, maxBackwardLimit);
dictionaryStart = Math.Min(position + 1, maxBackwardLimit);
var bestLenIn = features.HasFlag(Features.ExtensiveReferenceSearch) ? 0 : Math.Min(result.Copy.OutputLength - 1, maxLength);
var result2 = hasher.FindLongestMatch(position + 1, maxLength, maxDistance, dictionaryStart, builder.LastDistance, bestLenIn);
if (result2.Score >= result.Score + CostDiffLazy)
{
++position;
++insertLength;
result = result2;
if (++delayedBackwardReferencesInRow < 4 && position + hasher.HashTypeLength < posEnd)
{
--maxLength;
continue;
}
}
break;
}
var copy = result.Copy;
int len = copy.OutputLength;
applyRandomHeuristics = position + (2 * len) + LiteralSpreeLengthForSparseSearch;
copy.AddCommand(builder, Literal.FromBytes(input, position - insertLength, insertLength));
insertLength = 0;
int rangeStart = position + 2;
int rangeEnd = Math.Min(position + len, storeEnd);
if (copy is Copy.BackReference backReference && backReference.Distance < len >> 2)
{
rangeStart = Math.Min(rangeEnd, Math.Max(rangeStart, position + len - (backReference.Distance << 2)));
}
hasher.StoreRange(rangeStart, rangeEnd);
position += len;
}
else
{
++insertLength;
++position;
if (position > applyRandomHeuristics)
{
int skipStep;
int nextStopOffset;
if (position > applyRandomHeuristics + (4 * LiteralSpreeLengthForSparseSearch))
{
skipStep = 4;
nextStopOffset = 16;
}
else
{
skipStep = 2;
nextStopOffset = 8;
}
int margin = Math.Max(hasher.StoreLookahead - 1, skipStep);
int posJump = Math.Min(position + nextStopOffset, posEnd - margin);
while (position < posJump)
{
hasher.Store(position);
insertLength += skipStep;
position += skipStep;
}
}
}
}
insertLength += posEnd - position;
lastInsertLen = insertLength;
}
private NextStep Trawl(CompressedMetaBlockBuilder builder)
{
do
{
uint hash = nextHash;
int bytesBetweenHashLookups = (skip++) >> 5;
ip = nextIp;
nextIp = ip + bytesBetweenHashLookups;
if (nextIp > ipLimit)
{
return(NextStep.EmitRemainder);
}
nextHash = table.Hash(nextIp);
candidate = ip - lastDistance;
if (Match.Check(input, ip, candidate, MinMatchLen) && candidate < ip)
{
table[hash] = ip - baseIp;
break;
}
candidate = baseIp + table[hash];
table[hash] = ip - baseIp;
}while(!Match.Check(input, ip, candidate, MinMatchLen));
if (ip - candidate > SupportedWindowSize.Bytes)
{
return(NextStep.Trawl);
}
{
int @base = ip;
int matched = MinMatchLen + Match.DetermineLength(input, candidate + MinMatchLen, ip + MinMatchLen, ipEnd - ip - MinMatchLen);
int distance = @base - candidate;
int insert = @base - nextEmit;
ip += matched;
if (insert >= 6210 && ShouldUseUncompressedMode(insert))
{
outputTo = @base;
return(NextStep.OutputUncompressed);
}
if (distance == lastDistance)
{
builder.AddInsertCopy(Literal.FromBytes(input, nextEmit, insert), 2, DistanceInfo.ExplicitCodeZero);
}
else
{
builder.AddInsertCopy(Literal.FromBytes(input, nextEmit, insert), 2, distance);
lastDistance = distance;
}
builder.AddCopy(matched - 2, distance);
nextEmit = ip;
if (ip >= ipLimit)
{
return(NextStep.EmitRemainder);
}
candidate = table.UpdateAndGetCandidate(ip, baseIp);
}
while (Match.Check(input, ip, candidate, MinMatchLen))
{
int @base = ip;
int matched = MinMatchLen + Match.DetermineLength(input, candidate + MinMatchLen, ip + MinMatchLen, ipEnd - ip - MinMatchLen);
if (ip - candidate > SupportedWindowSize.Bytes)
{
break;
}
ip += matched;
lastDistance = @base - candidate;
builder.AddCopy(matched, lastDistance);
nextEmit = ip;
if (ip >= ipLimit)
{
return(NextStep.EmitRemainder);
}
candidate = table.UpdateAndGetCandidate(ip, baseIp);
}
nextHash = table.Hash(++ip);
return(NextStep.EmitCommandsNextHash);
}
private void TestBlockSplits(CompressedMetaBlockBuilder builder, BrotliCompressionParameters parameters, MetaBlockSizeTracker tracker, Category category)
{
int totalBlockLength = builder.GetTotalBlockLength(category);
if (totalBlockLength == 0)
{
return;
}
int step = Math.Max(1, (int)Math.Floor(Math.Log(totalBlockLength, 1.15)));
int stepTwice = step * 2;
if (totalBlockLength < stepTwice)
{
return;
}
var lengths = new List <int> {
totalBlockLength
};
var queue = new Queue <int>();
queue.Enqueue(0);
while (queue.TryDequeue(out int leftIndex))
{
if (lengths[leftIndex] < stepTwice)
{
continue;
}
int rightIndex = leftIndex + 1;
lengths.Insert(rightIndex, lengths[leftIndex] - step);
lengths[leftIndex] = step;
ApplyBlockSplit(builder.BlockTypes[category], lengths);
PrepareContextMap(builder, category, lengths.Count);
tracker.Test(builder, parameters);
while (lengths[leftIndex] + step < lengths[rightIndex] && lengths[rightIndex] >= stepTwice)
{
lengths[leftIndex] += step;
lengths[rightIndex] -= step;
ApplyBlockSplit(builder.BlockTypes[category], lengths);
PrepareContextMap(builder, category, lengths.Count);
tracker.Test(builder, parameters);
}
var smallest = tracker.Smallest;
if (smallest == null)
{
return;
}
var mb = smallest.Value.Item1;
int prev = lengths.Count;
lengths.Clear();
lengths.Add(mb.Header.BlockTypes[category].InitialLength);
lengths.AddRange(mb.Data.BlockSwitchCommands[category].Select(command => command.Length));
var finalLength = totalBlockLength - lengths.Sum();
if (finalLength > 0)
{
lengths.Add(totalBlockLength - lengths.Sum());
}
if (lengths.Count >= prev)
{
queue.Enqueue(leftIndex);
queue.Enqueue(rightIndex);
}
}
}
internal override int AddCommand(CompressedMetaBlockBuilder builder, IList <Literal> literals)
{
builder.AddInsertCopy(literals, in entry);
return(entry.OutputLength);
}
internal abstract int AddCommand(CompressedMetaBlockBuilder builder, IList <Literal> literals);
