public BrotliFileStreamTransformer(IBrotliFileStream original, BrotliCompressionParameters compressionParameters, params IBrotliTransformer[] transformers) { this.original = original; this.transformers = transformers; this.compressionParameters = compressionParameters; this.transformedQueue = new Queue <MetaBlock>(); }
public Simple(WindowSize windowSize, BrotliCompressionParameters compressionParameters, IBrotliEncoder encoder, params IBrotliTransformer[] transformers) { this.windowSize = windowSize; this.compressionParameters = compressionParameters; this.encoder = encoder; this.Transformers.AddRange(transformers); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { return(new CompressedMetaBlockBuilder(original, state) { DistanceParameters = DistanceParameters.Zero }.Build(parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { if (original.Data.InsertCopyCommands.Count <= 1 || original.Data.BlockSwitchCommands[Category.InsertCopy].Count > 0) { return(original, state); } var builder = new CompressedMetaBlockBuilder(original, state); var icBlocks = builder.BlockTypes[Category.InsertCopy]; var icCommands = builder.GetTotalBlockLength(Category.InsertCopy); icBlocks.SetInitialLength(icCommands / 2) .AddFinalBlock(1); return(builder.Build(parameters)); }
internal static (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters, IReadOnlyList <IBrotliTransformer> transformers) { var metaBlocks = new List <MetaBlock> { encodedMetaBlock }; var states = new List <BrotliGlobalState> { originalState }; foreach (var transformer in transformers) { var nextMetaBlocks = new List <MetaBlock>(); var nextStates = new List <BrotliGlobalState> { originalState }; // first meta-block starts with original state, second meta-block with the first meta-block's end state, etc. for (int index = 0; index < metaBlocks.Count; index++) { foreach (var(transformedMetaBlock, transformedState) in transformer.Transform(metaBlocks[index], states[index], compressionParameters)) { nextMetaBlocks.Add(transformedMetaBlock); nextStates.Add(transformedState); } } metaBlocks = nextMetaBlocks; states = nextStates; } return(metaBlocks, states[^ 1]);
private (IList <MetaBlock>, BrotliGlobalState) ApplyTransformerChain(BrotliGlobalState originalState, MetaBlock encodedMetaBlock, BrotliCompressionParameters compressionParameters) { return(ApplyTransformerChain(originalState, encodedMetaBlock, compressionParameters, Transformers)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var tracker = new MetaBlockSizeTracker(state); for (byte postfixBitCount = 0; postfixBitCount <= DistanceParameters.MaxPostfixBitCount; postfixBitCount++) { for (byte directCodeBits = 0; directCodeBits <= DistanceParameters.MaxDirectCodeBits; directCodeBits++) { builder.DistanceParameters = new DistanceParameters(postfixBitCount, directCodeBits); tracker.Test(builder, parameters, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]"); } } return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformCompressed(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { return(new [] { Transform(original, state, parameters) }); }
protected abstract (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters);
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state) { LiteralCtxMap = ContextMapBuilder.Literals.Simple, DistanceCtxMap = ContextMapBuilder.Distances.Simple }; foreach (var category in Categories.LID) { builder.BlockTypes[category].Reset(); } return(builder.UseSameLiteralContextMode(LiteralContextMode.LSB6).Build(parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var tracker = new MetaBlockSizeTracker(state); tracker.Test(builder, parameters); var blocker = new Blocker(parameters.DistanceCodePicker); parameters = new BrotliCompressionParameters.Builder(parameters) { DistanceCodePicker = blocker.Pick }.Build(); foreach (var code in DistanceCode.Last.Codes.Except(new DistanceCode[] { DistanceCode.Zero })) { var prev = tracker.SmallestSize; blocker.BlockedCodes.Add(code); tracker.Test(builder, parameters); if (tracker.SmallestSize < prev) { Debug.WriteLine("Blocking code " + code + " reduced size (" + prev + " > " + tracker.SmallestSize + "), keeping it..."); } else { Debug.WriteLine("Blocking code " + code + " did not improve the size, continuing..."); blocker.BlockedCodes.Remove(code); } } Debug.WriteLine("Final blocked codes: " + string.Join(", ", blocker.BlockedCodes)); return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
private BrotliEncodeInfo(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, BrotliGlobalState state, in ArraySegment <byte> bytes)
private void TestBlockSplits(CompressedMetaBlockBuilder builder, BrotliCompressionParameters parameters, MetaBlockSizeTracker tracker, Category category) { int totalBlockLength = builder.GetTotalBlockLength(category); if (totalBlockLength == 0) { return; } int step = Math.Max(1, (int)Math.Floor(Math.Log(totalBlockLength, 1.15))); int stepTwice = step * 2; if (totalBlockLength < stepTwice) { return; } var lengths = new List <int> { totalBlockLength }; var queue = new Queue <int>(); queue.Enqueue(0); while (queue.TryDequeue(out int leftIndex)) { if (lengths[leftIndex] < stepTwice) { continue; } int rightIndex = leftIndex + 1; lengths.Insert(rightIndex, lengths[leftIndex] - step); lengths[leftIndex] = step; ApplyBlockSplit(builder.BlockTypes[category], lengths); PrepareContextMap(builder, category, lengths.Count); tracker.Test(builder, parameters); while (lengths[leftIndex] + step < lengths[rightIndex] && lengths[rightIndex] >= stepTwice) { lengths[leftIndex] += step; lengths[rightIndex] -= step; ApplyBlockSplit(builder.BlockTypes[category], lengths); PrepareContextMap(builder, category, lengths.Count); tracker.Test(builder, parameters); } var smallest = tracker.Smallest; if (smallest == null) { return; } var mb = smallest.Value.Item1; int prev = lengths.Count; lengths.Clear(); lengths.Add(mb.Header.BlockTypes[category].InitialLength); lengths.AddRange(mb.Data.BlockSwitchCommands[category].Select(command => command.Length)); var finalLength = totalBlockLength - lengths.Sum(); if (finalLength > 0) { lengths.Add(totalBlockLength - lengths.Sum()); } if (lengths.Count >= prev) { queue.Enqueue(leftIndex); queue.Enqueue(rightIndex); } } }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state) { LiteralCtxMap = ContextMapBuilder.Literals.Simple, DistanceCtxMap = ContextMapBuilder.Distances.Simple }; foreach (var category in Categories.LID) { builder.BlockTypes[category].Reset(); } builder.UseSameLiteralContextMode(LiteralContextMode.UTF8); var tracker = new MetaBlockSizeTracker(state); tracker.Test(builder, parameters); foreach (var category in Categories.LID) { TestBlockSplits(builder, parameters, tracker, category); } tracker.Test(original); return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var uncompressed = new BrotliOutputStored(); state.AddOutputCallback(uncompressed); original.Decompress(state); state.RemoveOutputCallback(uncompressed); return(new MetaBlock.Uncompressed(uncompressed.AsBytes), state); }
protected sealed override IList <(MetaBlock MetaBlock, BrotliGlobalState NextState)> TransformLastEmpty(MetaBlock.LastEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters) { return(base.TransformLastEmpty(original, state, parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var literals = new List <Literal>(builder.GetTotalBlockLength(Category.Literal)); var lengthCodes = new List <InsertCopyLengthCode>(builder.GetTotalBlockLength(Category.InsertCopy)); var distanceCodes = new List <DistanceCode>(builder.GetTotalBlockLength(Category.Distance)); var distanceFreq = new FrequencyList <DistanceCode>(); var validDistanceCodes = new List <DistanceCode>(5); foreach (var command in original.Data.InsertCopyCommands) { literals.AddRange(command.Literals); state.OutputLiterals(command.Literals); if (command.CopyDistance == DistanceInfo.EndsAfterLiterals) { lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.PreferEnabled)); break; } if (!command.CopyDistance.FindCodes(original.Header.DistanceParameters, state, validDistanceCodes)) { lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.ForceEnabled)); } else { DistanceCode distanceCode; if (command.CopyDistance == DistanceInfo.ExplicitCodeZero) { distanceCode = DistanceCode.Zero; } else { distanceCode = validDistanceCodes.Count > 1 ? parameters.DistanceCodePicker(validDistanceCodes, distanceFreq) : validDistanceCodes[0]; } distanceFreq.Add(distanceCode); distanceCodes.Add(distanceCode); lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.Disable)); } } var origLitCtxMap = builder.LiteralCtxMap; if (origLitCtxMap.TreeCount == 1) { Split(builder, Category.Literal, literals, 512, 400.0); builder.UseSameLiteralContextMode(LiteralContextMode.UTF8); builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).RepeatFirstBlockType(true).Build(); } else { var literalContextMap = Enumerable.Range(0, origLitCtxMap.ContextsPerBlockType).Select(index => origLitCtxMap.DetermineTreeID(0, index)).ToArray(); var literalContextMode = builder.LiteralContextModes[0]; var literalBuffer = RingBufferFast <byte> .From(0, 0); Split(builder, Category.Literal, literals, 512, 400.0, new BlockSplitter <Literal> .ContextInfo(literalContextMap, literal => { literalBuffer.Push(literal.Value); return(literalContextMode.DetermineContextID(literalBuffer.Front, literalBuffer.Back)); })); builder.UseSameLiteralContextMode(literalContextMode); builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).Set(0, literalContextMap).RepeatFirstBlockType(true).Build(); } Split(builder, Category.InsertCopy, lengthCodes, 1024, 500.0); Split(builder, Category.Distance, distanceCodes, 512, 100.0); builder.DistanceCtxMap = new ContextMapBuilder.Distances(builder).RepeatFirstBlockType(true).Build(); return(builder.Build(parameters)); }
public BrotliFileStreamEncoder(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, byte[] bytes, IBrotliEncoder encoder) { this.encoder = encoder; this.encodeInfo = new BrotliEncodeInfo(fileParameters, compressionParameters, bytes); }