internal static (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters, IReadOnlyList <IBrotliTransformer> transformers) { var metaBlocks = new List <MetaBlock> { encodedMetaBlock }; var states = new List <BrotliGlobalState> { originalState }; foreach (var transformer in transformers) { var nextMetaBlocks = new List <MetaBlock>(); var nextStates = new List <BrotliGlobalState> { originalState }; // first meta-block starts with original state, second meta-block with the first meta-block's end state, etc. for (int index = 0; index < metaBlocks.Count; index++) { foreach (var(transformedMetaBlock, transformedState) in transformer.Transform(metaBlocks[index], states[index], compressionParameters)) { nextMetaBlocks.Add(transformedMetaBlock); nextStates.Add(transformedState); } } metaBlocks = nextMetaBlocks; states = nextStates; } return(metaBlocks, states[^ 1]);
public BrotliFileWriter(BrotliFileParameters fileParameters, BrotliSerializationParameters serializationParameters) { this.bits = new BitStream(); this.writer = bits.GetWriter(); this.FileParameters = fileParameters; this.SerializationParameters = serializationParameters; this.state = new BrotliGlobalState(FileParameters); WriteHeader(); }
public override void Decompress(BrotliGlobalState state) { foreach (InsertCopyCommand icCommand in Data.InsertCopyCommands) { state.OutputLiterals(icCommand.Literals); if (icCommand.CopyDistance != DistanceInfo.EndsAfterLiterals) { state.OutputCopy(icCommand.CopyLength, icCommand.CopyDistance); } } }
public static int GetValue(this DistanceInfo info, BrotliGlobalState state) { switch (info) { case DistanceInfo.EndsAfterLiterals: throw new InvalidOperationException("The command is missing a copy distance."); case DistanceInfo.ImplicitCodeZero: case DistanceInfo.ExplicitCodeZero: return(state.DistanceBuffer.Front); default: return(info >= DistanceInfo.FirstExactValue ? (int)info : throw new InvalidOperationException("Copy distance must be >= " + (int)DistanceInfo.FirstExactValue + ".")); } }
public CompressedMetaBlockBuilder(MetaBlock.Compressed metaBlock, BrotliGlobalState state) : this(state){ var header = metaBlock.Header; var data = metaBlock.Data; this.BlockTypes = header.BlockTypes.Select(info => new BlockSwitchBuilder(info, data.BlockSwitchCommands[info.Category])); this.DistanceParameters = header.DistanceParameters; this.LiteralContextModes = header.LiteralCtxModes.ToArray(); this.LiteralCtxMap = header.LiteralCtxMap; this.DistanceCtxMap = header.DistanceCtxMap; foreach (InsertCopyCommand command in data.InsertCopyCommands) { AddInsertCopyCommand(command); } }
public static bool CanEncodeUsing(this DistanceInfo info, DistanceCode code, BrotliGlobalState state) { switch (info) { case DistanceInfo.EndsAfterLiterals: case DistanceInfo.ImplicitCodeZero: return(false); case DistanceInfo.ExplicitCodeZero: return(code.Equals(DistanceCode.Zero)); default: return(code.CanEncodeValue(state, (int)info) && !code.Equals(DistanceCode.Zero)); } }
private BitStream Validate(BitStream bits) { IBrotliFileStream reader = BrotliFileReader.FromBytes(bits, Parameters.File.Dictionary); var output = new BrotliOutputStored(); var state = new BrotliGlobalState(reader.Parameters, output); reader.ForEachRemainingMetaBlock(metaBlock => metaBlock.Decompress(state)); if (!CollectionHelper.Equal(output.AsBytes, Uncompressed.Contents)) { throw new InvalidOperationException("Mismatched output bytes."); } return(bits); }
private static void ExtractMetadata(List <object> row, CompressedData data, BrotliGlobalState state) { row.Add(data.InsertCopyCommands.Count); int literalBytes = 0; int backRefBytes = 0; int backRefCount = 0; int dictRefBytes = 0; int dictRefCount = 0; foreach (var command in data.InsertCopyCommands) { var literals = command.Literals; var copyLength = command.CopyLength; var copyDistance = command.CopyDistance; state.OutputLiterals(literals); literalBytes += literals.Count; if (copyDistance == DistanceInfo.EndsAfterLiterals) { break; } var copy = state.OutputCopy(copyLength, copyDistance); if (copy.IsBackReference) { backRefBytes += copy.BytesWritten; ++backRefCount; } else { dictRefBytes += copy.BytesWritten; ++dictRefCount; } } row.Add(literalBytes); row.Add(backRefBytes); row.Add(dictRefBytes); row.Add(backRefCount); row.Add(dictRefCount); }
protected override IEnumerable <object[]> GenerateRows(BrotliFileGroup group, BrotliFile.Compressed file) { var quality = file.Identifier; var reader = file.Reader; BrotliGlobalState state = reader.State; MetaBlock? metaBlock; int index = 0; while ((metaBlock = reader.NextMetaBlock()) != null) { if (metaBlock is MetaBlock.Compressed c) { var row = new List <object> { file.Name, quality, index, metaBlock.DataLength.UncompressedBytes }; ExtractMetadata(row, c.Data, state); yield return(row.ToArray()); } ++index; state = reader.State; } }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { if (original.Data.InsertCopyCommands.Count <= 1 || original.Data.BlockSwitchCommands[Category.InsertCopy].Count > 0) { return(original, state); } var builder = new CompressedMetaBlockBuilder(original, state); var icBlocks = builder.BlockTypes[Category.InsertCopy]; var icCommands = builder.GetTotalBlockLength(Category.InsertCopy); icBlocks.SetInitialLength(icCommands / 2) .AddFinalBlock(1); return(builder.Build(parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state) { LiteralCtxMap = ContextMapBuilder.Literals.Simple, DistanceCtxMap = ContextMapBuilder.Distances.Simple }; foreach (var category in Categories.LID) { builder.BlockTypes[category].Reset(); } return(builder.UseSameLiteralContextMode(LiteralContextMode.LSB6).Build(parameters)); }
// Construction public CompressedMetaBlockBuilder(BrotliGlobalState state) { this.initialState = state.Clone(); this.intermediateState = state.Clone(); }
public override void Decompress(BrotliGlobalState state) { state.OutputBytes(uncompressedData); }
private (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters) { return(ApplyTransformerChain(originalState, encodedMetaBlock, compressionParameters, Transformers)); }
public ReadContext(BrotliGlobalState state, DataLength dataLength) { this.State = state; this.DataLength = dataLength; }
protected override int ReadValue(BrotliGlobalState state, IBitReader reader) { return(state.DistanceBuffer[index] + offset); }
public abstract void Decompress(BrotliGlobalState state);
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var literals = new List <Literal>(builder.GetTotalBlockLength(Category.Literal)); var lengthCodes = new List <InsertCopyLengthCode>(builder.GetTotalBlockLength(Category.InsertCopy)); var distanceCodes = new List <DistanceCode>(builder.GetTotalBlockLength(Category.Distance)); var distanceFreq = new FrequencyList <DistanceCode>(); var validDistanceCodes = new List <DistanceCode>(5); foreach (var command in original.Data.InsertCopyCommands) { literals.AddRange(command.Literals); state.OutputLiterals(command.Literals); if (command.CopyDistance == DistanceInfo.EndsAfterLiterals) { lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.PreferEnabled)); break; } if (!command.CopyDistance.FindCodes(original.Header.DistanceParameters, state, validDistanceCodes)) { lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.ForceEnabled)); } else { DistanceCode distanceCode; if (command.CopyDistance == DistanceInfo.ExplicitCodeZero) { distanceCode = DistanceCode.Zero; } else { distanceCode = validDistanceCodes.Count > 1 ? parameters.DistanceCodePicker(validDistanceCodes, distanceFreq) : validDistanceCodes[0]; } distanceFreq.Add(distanceCode); distanceCodes.Add(distanceCode); lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.Disable)); } } var origLitCtxMap = builder.LiteralCtxMap; if (origLitCtxMap.TreeCount == 1) { Split(builder, Category.Literal, literals, 512, 400.0); builder.UseSameLiteralContextMode(LiteralContextMode.UTF8); builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).RepeatFirstBlockType(true).Build(); } else { var literalContextMap = Enumerable.Range(0, origLitCtxMap.ContextsPerBlockType).Select(index => origLitCtxMap.DetermineTreeID(0, index)).ToArray(); var literalContextMode = builder.LiteralContextModes[0]; var literalBuffer = RingBufferFast <byte> .From(0, 0); Split(builder, Category.Literal, literals, 512, 400.0, new BlockSplitter <Literal> .ContextInfo(literalContextMap, literal => { literalBuffer.Push(literal.Value); return(literalContextMode.DetermineContextID(literalBuffer.Front, literalBuffer.Back)); })); builder.UseSameLiteralContextMode(literalContextMode); builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).Set(0, literalContextMap).RepeatFirstBlockType(true).Build(); } Split(builder, Category.InsertCopy, lengthCodes, 1024, 500.0); Split(builder, Category.Distance, distanceCodes, 512, 100.0); builder.DistanceCtxMap = new ContextMapBuilder.Distances(builder).RepeatFirstBlockType(true).Build(); return(builder.Build(parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var uncompressed = new BrotliOutputStored(); state.AddOutputCallback(uncompressed); original.Decompress(state); state.RemoveOutputCallback(uncompressed); return(new MetaBlock.Uncompressed(uncompressed.AsBytes), state); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var tracker = new MetaBlockSizeTracker(state); for (byte postfixBitCount = 0; postfixBitCount <= DistanceParameters.MaxPostfixBitCount; postfixBitCount++) { for (byte directCodeBits = 0; directCodeBits <= DistanceParameters.MaxDirectCodeBits; directCodeBits++) { builder.DistanceParameters = new DistanceParameters(postfixBitCount, directCodeBits); tracker.Test(builder, parameters, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]"); } } return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state) { LiteralCtxMap = ContextMapBuilder.Literals.Simple, DistanceCtxMap = ContextMapBuilder.Distances.Simple }; foreach (var category in Categories.LID) { builder.BlockTypes[category].Reset(); } builder.UseSameLiteralContextMode(LiteralContextMode.UTF8); var tracker = new MetaBlockSizeTracker(state); tracker.Test(builder, parameters); foreach (var category in Categories.LID) { TestBlockSplits(builder, parameters, tracker, category); } tracker.Test(original); return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
private BrotliEncodeInfo(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, BrotliGlobalState state, in ArraySegment <byte> bytes)
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { return(new CompressedMetaBlockBuilder(original, state) { DistanceParameters = DistanceParameters.Zero }.Build(parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var tracker = new MetaBlockSizeTracker(state); tracker.Test(builder, parameters); var blocker = new Blocker(parameters.DistanceCodePicker); parameters = new BrotliCompressionParameters.Builder(parameters) { DistanceCodePicker = blocker.Pick }.Build(); foreach (var code in DistanceCode.Last.Codes.Except(new DistanceCode[] { DistanceCode.Zero })) { var prev = tracker.SmallestSize; blocker.BlockedCodes.Add(code); tracker.Test(builder, parameters); if (tracker.SmallestSize < prev) { Debug.WriteLine("Blocking code " + code + " reduced size (" + prev + " > " + tracker.SmallestSize + "), keeping it..."); } else { Debug.WriteLine("Blocking code " + code + " did not improve the size, continuing..."); blocker.BlockedCodes.Remove(code); } } Debug.WriteLine("Final blocked codes: " + string.Join(", ", blocker.BlockedCodes)); return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
public override void Decompress(BrotliGlobalState state) { }
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformCompressed(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { return(new [] { Transform(original, state, parameters) }); }
public override bool CanEncodeValue(BrotliGlobalState state, int value) { return(value == state.DistanceBuffer[index] + offset); }
protected abstract (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters);
protected override void WriteValue(BrotliGlobalState state, int value, IBitWriter writer) { // no extra bits }
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformLastEmpty(MetaBlock.LastEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters) { return(base.TransformLastEmpty(original, state, parameters)); }