internal static (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters, IReadOnlyList <IBrotliTransformer> transformers)
{
var metaBlocks = new List <MetaBlock> {
encodedMetaBlock
};
var states = new List <BrotliGlobalState> {
originalState
};
foreach (var transformer in transformers)
{
var nextMetaBlocks = new List <MetaBlock>();
var nextStates = new List <BrotliGlobalState> {
originalState
}; // first meta-block starts with original state, second meta-block with the first meta-block's end state, etc.
for (int index = 0; index < metaBlocks.Count; index++)
{
foreach (var(transformedMetaBlock, transformedState) in transformer.Transform(metaBlocks[index], states[index], compressionParameters))
{
nextMetaBlocks.Add(transformedMetaBlock);
nextStates.Add(transformedState);
}
}
metaBlocks = nextMetaBlocks;
states = nextStates;
}
return(metaBlocks, states[^ 1]);
public BrotliFileWriter(BrotliFileParameters fileParameters, BrotliSerializationParameters serializationParameters)
{
this.bits = new BitStream();
this.writer = bits.GetWriter();
this.FileParameters = fileParameters;
this.SerializationParameters = serializationParameters;
this.state = new BrotliGlobalState(FileParameters);
WriteHeader();
}
public override void Decompress(BrotliGlobalState state)
{
foreach (InsertCopyCommand icCommand in Data.InsertCopyCommands)
{
state.OutputLiterals(icCommand.Literals);
if (icCommand.CopyDistance != DistanceInfo.EndsAfterLiterals)
{
state.OutputCopy(icCommand.CopyLength, icCommand.CopyDistance);
}
}
}
public static int GetValue(this DistanceInfo info, BrotliGlobalState state)
{
switch (info)
{
case DistanceInfo.EndsAfterLiterals:
throw new InvalidOperationException("The command is missing a copy distance.");
case DistanceInfo.ImplicitCodeZero:
case DistanceInfo.ExplicitCodeZero:
return(state.DistanceBuffer.Front);
default:
return(info >= DistanceInfo.FirstExactValue ? (int)info : throw new InvalidOperationException("Copy distance must be >= " + (int)DistanceInfo.FirstExactValue + "."));
}
}
public CompressedMetaBlockBuilder(MetaBlock.Compressed metaBlock, BrotliGlobalState state) : this(state){
var header = metaBlock.Header;
var data = metaBlock.Data;
this.BlockTypes = header.BlockTypes.Select(info => new BlockSwitchBuilder(info, data.BlockSwitchCommands[info.Category]));
this.DistanceParameters = header.DistanceParameters;
this.LiteralContextModes = header.LiteralCtxModes.ToArray();
this.LiteralCtxMap = header.LiteralCtxMap;
this.DistanceCtxMap = header.DistanceCtxMap;
foreach (InsertCopyCommand command in data.InsertCopyCommands)
{
AddInsertCopyCommand(command);
}
}
public static bool CanEncodeUsing(this DistanceInfo info, DistanceCode code, BrotliGlobalState state)
{
switch (info)
{
case DistanceInfo.EndsAfterLiterals:
case DistanceInfo.ImplicitCodeZero:
return(false);
case DistanceInfo.ExplicitCodeZero:
return(code.Equals(DistanceCode.Zero));
default:
return(code.CanEncodeValue(state, (int)info) && !code.Equals(DistanceCode.Zero));
}
}
private BitStream Validate(BitStream bits)
{
IBrotliFileStream reader = BrotliFileReader.FromBytes(bits, Parameters.File.Dictionary);
var output = new BrotliOutputStored();
var state = new BrotliGlobalState(reader.Parameters, output);
reader.ForEachRemainingMetaBlock(metaBlock => metaBlock.Decompress(state));
if (!CollectionHelper.Equal(output.AsBytes, Uncompressed.Contents))
{
throw new InvalidOperationException("Mismatched output bytes.");
}
return(bits);
}
private static void ExtractMetadata(List <object> row, CompressedData data, BrotliGlobalState state)
{
row.Add(data.InsertCopyCommands.Count);
int literalBytes = 0;
int backRefBytes = 0;
int backRefCount = 0;
int dictRefBytes = 0;
int dictRefCount = 0;
foreach (var command in data.InsertCopyCommands)
{
var literals = command.Literals;
var copyLength = command.CopyLength;
var copyDistance = command.CopyDistance;
state.OutputLiterals(literals);
literalBytes += literals.Count;
if (copyDistance == DistanceInfo.EndsAfterLiterals)
{
break;
}
var copy = state.OutputCopy(copyLength, copyDistance);
if (copy.IsBackReference)
{
backRefBytes += copy.BytesWritten;
++backRefCount;
}
else
{
dictRefBytes += copy.BytesWritten;
++dictRefCount;
}
}
row.Add(literalBytes);
row.Add(backRefBytes);
row.Add(dictRefBytes);
row.Add(backRefCount);
row.Add(dictRefCount);
}
protected override IEnumerable <object[]> GenerateRows(BrotliFileGroup group, BrotliFile.Compressed file)
{
var quality = file.Identifier;
var reader = file.Reader;
BrotliGlobalState state = reader.State;
MetaBlock? metaBlock;
int index = 0;
while ((metaBlock = reader.NextMetaBlock()) != null)
{
if (metaBlock is MetaBlock.Compressed c)
{
var row = new List <object> {
file.Name, quality, index, metaBlock.DataLength.UncompressedBytes
};
ExtractMetadata(row, c.Data, state);
yield return(row.ToArray());
}
++index;
state = reader.State;
}
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
if (original.Data.InsertCopyCommands.Count <= 1 || original.Data.BlockSwitchCommands[Category.InsertCopy].Count > 0)
{
return(original, state);
}
var builder = new CompressedMetaBlockBuilder(original, state);
var icBlocks = builder.BlockTypes[Category.InsertCopy];
var icCommands = builder.GetTotalBlockLength(Category.InsertCopy);
icBlocks.SetInitialLength(icCommands / 2)
.AddFinalBlock(1);
return(builder.Build(parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state)
{
LiteralCtxMap = ContextMapBuilder.Literals.Simple,
DistanceCtxMap = ContextMapBuilder.Distances.Simple
};
foreach (var category in Categories.LID)
{
builder.BlockTypes[category].Reset();
}
return(builder.UseSameLiteralContextMode(LiteralContextMode.LSB6).Build(parameters));
}
// Construction
public CompressedMetaBlockBuilder(BrotliGlobalState state)
{
this.initialState = state.Clone();
this.intermediateState = state.Clone();
}
public override void Decompress(BrotliGlobalState state)
{
state.OutputBytes(uncompressedData);
}
private (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters)
{
return(ApplyTransformerChain(originalState, encodedMetaBlock, compressionParameters, Transformers));
}
public ReadContext(BrotliGlobalState state, DataLength dataLength)
{
this.State = state;
this.DataLength = dataLength;
}
protected override int ReadValue(BrotliGlobalState state, IBitReader reader)
{
return(state.DistanceBuffer[index] + offset);
}
public abstract void Decompress(BrotliGlobalState state);
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var literals = new List <Literal>(builder.GetTotalBlockLength(Category.Literal));
var lengthCodes = new List <InsertCopyLengthCode>(builder.GetTotalBlockLength(Category.InsertCopy));
var distanceCodes = new List <DistanceCode>(builder.GetTotalBlockLength(Category.Distance));
var distanceFreq = new FrequencyList <DistanceCode>();
var validDistanceCodes = new List <DistanceCode>(5);
foreach (var command in original.Data.InsertCopyCommands)
{
literals.AddRange(command.Literals);
state.OutputLiterals(command.Literals);
if (command.CopyDistance == DistanceInfo.EndsAfterLiterals)
{
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.PreferEnabled));
break;
}
if (!command.CopyDistance.FindCodes(original.Header.DistanceParameters, state, validDistanceCodes))
{
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.ForceEnabled));
}
else
{
DistanceCode distanceCode;
if (command.CopyDistance == DistanceInfo.ExplicitCodeZero)
{
distanceCode = DistanceCode.Zero;
}
else
{
distanceCode = validDistanceCodes.Count > 1 ? parameters.DistanceCodePicker(validDistanceCodes, distanceFreq) : validDistanceCodes[0];
}
distanceFreq.Add(distanceCode);
distanceCodes.Add(distanceCode);
lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.Disable));
}
}
var origLitCtxMap = builder.LiteralCtxMap;
if (origLitCtxMap.TreeCount == 1)
{
Split(builder, Category.Literal, literals, 512, 400.0);
builder.UseSameLiteralContextMode(LiteralContextMode.UTF8);
builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).RepeatFirstBlockType(true).Build();
}
else
{
var literalContextMap = Enumerable.Range(0, origLitCtxMap.ContextsPerBlockType).Select(index => origLitCtxMap.DetermineTreeID(0, index)).ToArray();
var literalContextMode = builder.LiteralContextModes[0];
var literalBuffer = RingBufferFast <byte> .From(0, 0);
Split(builder, Category.Literal, literals, 512, 400.0, new BlockSplitter <Literal> .ContextInfo(literalContextMap, literal => {
literalBuffer.Push(literal.Value);
return(literalContextMode.DetermineContextID(literalBuffer.Front, literalBuffer.Back));
}));
builder.UseSameLiteralContextMode(literalContextMode);
builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).Set(0, literalContextMap).RepeatFirstBlockType(true).Build();
}
Split(builder, Category.InsertCopy, lengthCodes, 1024, 500.0);
Split(builder, Category.Distance, distanceCodes, 512, 100.0);
builder.DistanceCtxMap = new ContextMapBuilder.Distances(builder).RepeatFirstBlockType(true).Build();
return(builder.Build(parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var uncompressed = new BrotliOutputStored();
state.AddOutputCallback(uncompressed);
original.Decompress(state);
state.RemoveOutputCallback(uncompressed);
return(new MetaBlock.Uncompressed(uncompressed.AsBytes), state);
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var tracker = new MetaBlockSizeTracker(state);
for (byte postfixBitCount = 0; postfixBitCount <= DistanceParameters.MaxPostfixBitCount; postfixBitCount++)
{
for (byte directCodeBits = 0; directCodeBits <= DistanceParameters.MaxDirectCodeBits; directCodeBits++)
{
builder.DistanceParameters = new DistanceParameters(postfixBitCount, directCodeBits);
tracker.Test(builder, parameters, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]");
}
}
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state)
{
LiteralCtxMap = ContextMapBuilder.Literals.Simple,
DistanceCtxMap = ContextMapBuilder.Distances.Simple
};
foreach (var category in Categories.LID)
{
builder.BlockTypes[category].Reset();
}
builder.UseSameLiteralContextMode(LiteralContextMode.UTF8);
var tracker = new MetaBlockSizeTracker(state);
tracker.Test(builder, parameters);
foreach (var category in Categories.LID)
{
TestBlockSplits(builder, parameters, tracker, category);
}
tracker.Test(original);
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
private BrotliEncodeInfo(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, BrotliGlobalState state, in ArraySegment <byte> bytes)
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
return(new CompressedMetaBlockBuilder(original, state)
{
DistanceParameters = DistanceParameters.Zero
}.Build(parameters));
}
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
var builder = new CompressedMetaBlockBuilder(original, state);
var tracker = new MetaBlockSizeTracker(state);
tracker.Test(builder, parameters);
var blocker = new Blocker(parameters.DistanceCodePicker);
parameters = new BrotliCompressionParameters.Builder(parameters)
{
DistanceCodePicker = blocker.Pick
}.Build();
foreach (var code in DistanceCode.Last.Codes.Except(new DistanceCode[] { DistanceCode.Zero }))
{
var prev = tracker.SmallestSize;
blocker.BlockedCodes.Add(code);
tracker.Test(builder, parameters);
if (tracker.SmallestSize < prev)
{
Debug.WriteLine("Blocking code " + code + " reduced size (" + prev + " > " + tracker.SmallestSize + "), keeping it...");
}
else
{
Debug.WriteLine("Blocking code " + code + " did not improve the size, continuing...");
blocker.BlockedCodes.Remove(code);
}
}
Debug.WriteLine("Final blocked codes: " + string.Join(", ", blocker.BlockedCodes));
return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks."));
}
public override void Decompress(BrotliGlobalState state)
{
}
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformCompressed(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
return(new [] { Transform(original, state, parameters) });
}
public override bool CanEncodeValue(BrotliGlobalState state, int value)
{
return(value == state.DistanceBuffer[index] + offset);
}
protected abstract (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters);
protected override void WriteValue(BrotliGlobalState state, int value, IBitWriter writer)
{
// no extra bits
}
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformLastEmpty(MetaBlock.LastEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters)
{
return(base.TransformLastEmpty(original, state, parameters));
}
