public BrotliFileStreamTransformer(IBrotliFileStream original, BrotliCompressionParameters compressionParameters, params IBrotliTransformer[] transformers)
{
this.original = original;
this.transformers = transformers;
this.compressionParameters = compressionParameters;
this.transformedQueue = new Queue <MetaBlock>();
}
public Simple(WindowSize windowSize, BrotliCompressionParameters compressionParameters, IBrotliEncoder encoder, params IBrotliTransformer[] transformers)
{
this.windowSize = windowSize;
this.compressionParameters = compressionParameters;
this.encoder = encoder;
this.Transformers.AddRange(transformers);
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
return(new CompressedMetaBlockBuilder(original, state)
{
DistanceParameters = DistanceParameters.Zero
}.Build(parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
if (original.Data.InsertCopyCommands.Count <= 1 || original.Data.BlockSwitchCommands[Category.InsertCopy].Count > 0)
{
return(original, state);
}
var builder = new CompressedMetaBlockBuilder(original, state);
var icBlocks = builder.BlockTypes[Category.InsertCopy];
var icCommands = builder.GetTotalBlockLength(Category.InsertCopy);
icBlocks.SetInitialLength(icCommands / 2)
.AddFinalBlock(1);
return(builder.Build(parameters));
}
internal static (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters, IReadOnlyList <IBrotliTransformer> transformers)
{
var metaBlocks = new List <MetaBlock> {
encodedMetaBlock
};
var states = new List <BrotliGlobalState> {
originalState
};
foreach (var transformer in transformers)
{
var nextMetaBlocks = new List <MetaBlock>();
var nextStates = new List <BrotliGlobalState> {
originalState
}; // first meta-block starts with original state, second meta-block with the first meta-block's end state, etc.
for (int index = 0; index < metaBlocks.Count; index++)
{
foreach (var(transformedMetaBlock, transformedState) in transformer.Transform(metaBlocks[index], states[index], compressionParameters))
{
nextMetaBlocks.Add(transformedMetaBlock);
nextStates.Add(transformedState);
}
}
metaBlocks = nextMetaBlocks;
states = nextStates;
}
return(metaBlocks, states[^ 1]);
private (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters)
{
return(ApplyTransformerChain(originalState, encodedMetaBlock, compressionParameters, Transformers));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var tracker = new MetaBlockSizeTracker(state);
for (byte postfixBitCount = 0; postfixBitCount <= DistanceParameters.MaxPostfixBitCount; postfixBitCount++)
{
for (byte directCodeBits = 0; directCodeBits <= DistanceParameters.MaxDirectCodeBits; directCodeBits++)
{
builder.DistanceParameters = new DistanceParameters(postfixBitCount, directCodeBits);
tracker.Test(builder, parameters, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]");
}
}
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformCompressed(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
return(new [] { Transform(original, state, parameters) });
}
protected abstract (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters);
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state)
{
LiteralCtxMap = ContextMapBuilder.Literals.Simple,
DistanceCtxMap = ContextMapBuilder.Distances.Simple
};
foreach (var category in Categories.LID)
{
builder.BlockTypes[category].Reset();
}
return(builder.UseSameLiteralContextMode(LiteralContextMode.LSB6).Build(parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var tracker = new MetaBlockSizeTracker(state);
tracker.Test(builder, parameters);
var blocker = new Blocker(parameters.DistanceCodePicker);
parameters = new BrotliCompressionParameters.Builder(parameters)
{
DistanceCodePicker = blocker.Pick
}.Build();
foreach (var code in DistanceCode.Last.Codes.Except(new DistanceCode[] { DistanceCode.Zero }))
{
var prev = tracker.SmallestSize;
blocker.BlockedCodes.Add(code);
tracker.Test(builder, parameters);
if (tracker.SmallestSize < prev)
{
Debug.WriteLine("Blocking code " + code + " reduced size (" + prev + " > " + tracker.SmallestSize + "), keeping it...");
}
else
{
Debug.WriteLine("Blocking code " + code + " did not improve the size, continuing...");
blocker.BlockedCodes.Remove(code);
}
}
Debug.WriteLine("Final blocked codes: " + string.Join(", ", blocker.BlockedCodes));
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
private BrotliEncodeInfo(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, BrotliGlobalState state, in ArraySegment <byte> bytes)
private void TestBlockSplits(CompressedMetaBlockBuilder builder, BrotliCompressionParameters parameters, MetaBlockSizeTracker tracker, Category category)
{
int totalBlockLength = builder.GetTotalBlockLength(category);
if (totalBlockLength == 0)
{
return;
}
int step = Math.Max(1, (int)Math.Floor(Math.Log(totalBlockLength, 1.15)));
int stepTwice = step * 2;
if (totalBlockLength < stepTwice)
{
return;
}
var lengths = new List <int> {
totalBlockLength
};
var queue = new Queue <int>();
queue.Enqueue(0);
while (queue.TryDequeue(out int leftIndex))
{
if (lengths[leftIndex] < stepTwice)
{
continue;
}
int rightIndex = leftIndex + 1;
lengths.Insert(rightIndex, lengths[leftIndex] - step);
lengths[leftIndex] = step;
ApplyBlockSplit(builder.BlockTypes[category], lengths);
PrepareContextMap(builder, category, lengths.Count);
tracker.Test(builder, parameters);
while (lengths[leftIndex] + step < lengths[rightIndex] && lengths[rightIndex] >= stepTwice)
{
lengths[leftIndex] += step;
lengths[rightIndex] -= step;
ApplyBlockSplit(builder.BlockTypes[category], lengths);
PrepareContextMap(builder, category, lengths.Count);
tracker.Test(builder, parameters);
}
var smallest = tracker.Smallest;
if (smallest == null)
{
return;
}
var mb = smallest.Value.Item1;
int prev = lengths.Count;
lengths.Clear();
lengths.Add(mb.Header.BlockTypes[category].InitialLength);
lengths.AddRange(mb.Data.BlockSwitchCommands[category].Select(command => command.Length));
var finalLength = totalBlockLength - lengths.Sum();
if (finalLength > 0)
{
lengths.Add(totalBlockLength - lengths.Sum());
}
if (lengths.Count >= prev)
{
queue.Enqueue(leftIndex);
queue.Enqueue(rightIndex);
}
}
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state)
{
LiteralCtxMap = ContextMapBuilder.Literals.Simple,
DistanceCtxMap = ContextMapBuilder.Distances.Simple
};
foreach (var category in Categories.LID)
{
builder.BlockTypes[category].Reset();
}
builder.UseSameLiteralContextMode(LiteralContextMode.UTF8);
var tracker = new MetaBlockSizeTracker(state);
tracker.Test(builder, parameters);
foreach (var category in Categories.LID)
{
TestBlockSplits(builder, parameters, tracker, category);
}
tracker.Test(original);
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var uncompressed = new BrotliOutputStored();
state.AddOutputCallback(uncompressed);
original.Decompress(state);
state.RemoveOutputCallback(uncompressed);
return(new MetaBlock.Uncompressed(uncompressed.AsBytes), state);
}
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformLastEmpty(MetaBlock.LastEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
return(base.TransformLastEmpty(original, state, parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var literals = new List <Literal>(builder.GetTotalBlockLength(Category.Literal));
var lengthCodes = new List <InsertCopyLengthCode>(builder.GetTotalBlockLength(Category.InsertCopy));
var distanceCodes = new List <DistanceCode>(builder.GetTotalBlockLength(Category.Distance));
var distanceFreq = new FrequencyList <DistanceCode>();
var validDistanceCodes = new List <DistanceCode>(5);
foreach (var command in original.Data.InsertCopyCommands)
{
literals.AddRange(command.Literals);
state.OutputLiterals(command.Literals);
if (command.CopyDistance == DistanceInfo.EndsAfterLiterals)
{
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.PreferEnabled));
break;
}
if (!command.CopyDistance.FindCodes(original.Header.DistanceParameters, state, validDistanceCodes))
{
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.ForceEnabled));
}
else
{
DistanceCode distanceCode;
if (command.CopyDistance == DistanceInfo.ExplicitCodeZero)
{
distanceCode = DistanceCode.Zero;
}
else
{
distanceCode = validDistanceCodes.Count > 1 ? parameters.DistanceCodePicker(validDistanceCodes, distanceFreq) : validDistanceCodes[0];
}
distanceFreq.Add(distanceCode);
distanceCodes.Add(distanceCode);
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.Disable));
}
}
var origLitCtxMap = builder.LiteralCtxMap;
if (origLitCtxMap.TreeCount == 1)
{
Split(builder, Category.Literal, literals, 512, 400.0);
builder.UseSameLiteralContextMode(LiteralContextMode.UTF8);
builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).RepeatFirstBlockType(true).Build();
}
else
{
var literalContextMap = Enumerable.Range(0, origLitCtxMap.ContextsPerBlockType).Select(index => origLitCtxMap.DetermineTreeID(0, index)).ToArray();
var literalContextMode = builder.LiteralContextModes[0];
var literalBuffer = RingBufferFast <byte> .From(0, 0);
Split(builder, Category.Literal, literals, 512, 400.0, new BlockSplitter <Literal> .ContextInfo(literalContextMap, literal => {
literalBuffer.Push(literal.Value);
return(literalContextMode.DetermineContextID(literalBuffer.Front, literalBuffer.Back));
}));
builder.UseSameLiteralContextMode(literalContextMode);
builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).Set(0, literalContextMap).RepeatFirstBlockType(true).Build();
}
Split(builder, Category.InsertCopy, lengthCodes, 1024, 500.0);
Split(builder, Category.Distance, distanceCodes, 512, 100.0);
builder.DistanceCtxMap = new ContextMapBuilder.Distances(builder).RepeatFirstBlockType(true).Build();
return(builder.Build(parameters));
}
public BrotliFileStreamEncoder(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, byte[] bytes, IBrotliEncoder encoder)
{
this.encoder = encoder;
this.encodeInfo = new BrotliEncodeInfo(fileParameters, compressionParameters, bytes);
}
