protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state) { LiteralCtxMap = ContextMapBuilder.Literals.Simple, DistanceCtxMap = ContextMapBuilder.Distances.Simple }; foreach (var category in Categories.LID) { builder.BlockTypes[category].Reset(); } builder.UseSameLiteralContextMode(LiteralContextMode.UTF8); var tracker = new MetaBlockSizeTracker(state); tracker.Test(builder, parameters); foreach (var category in Categories.LID) { TestBlockSplits(builder, parameters, tracker, category); } tracker.Test(original); return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
private NextStep EmitRemainder(CompressedMetaBlockBuilder builder) { nextIpStart += blockSize; inputSize -= blockSize; blockSize = Math.Min(inputSize, MergeBlockSize); if (inputSize > 0 && totalBlockSize + blockSize <= (1 << 20) && ShouldMergeBlock(nextIpStart, blockSize)) { totalBlockSize += blockSize; return(NextStep.EmitCommands); } if (nextEmit < ipEnd) { int insert = ipEnd - nextEmit; if (insert >= 6210 && ShouldUseUncompressedMode(insert)) { outputTo = ipEnd; return(NextStep.OutputUncompressed); } else { builder.AddInsert(Literal.FromBytes(input, nextEmit, insert)); } } nextEmit = ipEnd; return(NextStep.OutputCompressed); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state) { LiteralCtxMap = ContextMapBuilder.Literals.Simple, DistanceCtxMap = ContextMapBuilder.Distances.Simple }; foreach (var category in Categories.LID) { builder.BlockTypes[category].Reset(); } return(builder.UseSameLiteralContextMode(LiteralContextMode.LSB6).Build(parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { if (original.Data.InsertCopyCommands.Count <= 1 || original.Data.BlockSwitchCommands[Category.InsertCopy].Count > 0) { return(original, state); } var builder = new CompressedMetaBlockBuilder(original, state); var icBlocks = builder.BlockTypes[Category.InsertCopy]; var icCommands = builder.GetTotalBlockLength(Category.InsertCopy); icBlocks.SetInitialLength(icCommands / 2) .AddFinalBlock(1); return(builder.Build(parameters)); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var tracker = new MetaBlockSizeTracker(state); for (byte postfixBitCount = 0; postfixBitCount <= DistanceParameters.MaxPostfixBitCount; postfixBitCount++) { for (byte directCodeBits = 0; directCodeBits <= DistanceParameters.MaxDirectCodeBits; directCodeBits++) { builder.DistanceParameters = new DistanceParameters(postfixBitCount, directCodeBits); tracker.Test(builder, parameters, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]"); } } return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
private void PrepareContextMap(CompressedMetaBlockBuilder builder, Category category, int lengthSplits) { int blockTypes = Math.Min(lengthSplits, 256); if (category == Category.Literal) { if (builder.LiteralCtxMap.BlockTypes != blockTypes) { builder.UseSameLiteralContextMode(LiteralContextMode.UTF8); builder.LiteralCtxMap = new ContextMapBuilder.Literals(blockTypes).RepeatFirstBlockType(lengthSplits <= blockTypes).Build(); } } else if (category == Category.Distance) { if (builder.DistanceCtxMap.BlockTypes != blockTypes) { builder.DistanceCtxMap = new ContextMapBuilder.Distances(blockTypes).RepeatFirstBlockType(lengthSplits <= blockTypes).Build(); } } }
/// <summary> /// Adapted from https://github.com/google/brotli/blob/master/c/enc/encode.c (EncodeData). /// </summary> private bool ShouldContinueThisBlock(CompressedMetaBlockBuilder builder) { int maxLength = MaxMetaBlockSize(fileParameters, lgBlock); if (builder.OutputSize + InputBlockSize(lgBlock) > maxLength) { return(false); } int totalCommands = builder.InsertCopyCommands.Count; int totalLiterals = builder.InsertCopyCommands.Sum(icCommand => icCommand.Literals.Count); if (!features.HasFlag(Features.BlockSplit) && totalCommands + totalLiterals >= 0x2FFF /* 12287 */) { return(false); } int maxCommands = maxLength / 8; int maxLiterals = maxLength / 8; return(totalCommands < maxCommands && totalLiterals < maxLiterals); }
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var tracker = new MetaBlockSizeTracker(state); tracker.Test(builder, parameters); var blocker = new Blocker(parameters.DistanceCodePicker); parameters = new BrotliCompressionParameters.Builder(parameters) { DistanceCodePicker = blocker.Pick }.Build(); foreach (var code in DistanceCode.Last.Codes.Except(new DistanceCode[] { DistanceCode.Zero })) { var prev = tracker.SmallestSize; blocker.BlockedCodes.Add(code); tracker.Test(builder, parameters); if (tracker.SmallestSize < prev) { Debug.WriteLine("Blocking code " + code + " reduced size (" + prev + " > " + tracker.SmallestSize + "), keeping it..."); } else { Debug.WriteLine("Blocking code " + code + " did not improve the size, continuing..."); blocker.BlockedCodes.Remove(code); } } Debug.WriteLine("Final blocked codes: " + string.Join(", ", blocker.BlockedCodes)); return(tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.")); }
private void Split <T>(CompressedMetaBlockBuilder builder, Category category, List <T> sequence, int minBlockSize, double splitThreshold, in BlockSplitter <T> .ContextInfo?context = null) where T : IComparable <T>
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters) { var builder = new CompressedMetaBlockBuilder(original, state); var literals = new List <Literal>(builder.GetTotalBlockLength(Category.Literal)); var lengthCodes = new List <InsertCopyLengthCode>(builder.GetTotalBlockLength(Category.InsertCopy)); var distanceCodes = new List <DistanceCode>(builder.GetTotalBlockLength(Category.Distance)); var distanceFreq = new FrequencyList <DistanceCode>(); var validDistanceCodes = new List <DistanceCode>(5); foreach (var command in original.Data.InsertCopyCommands) { literals.AddRange(command.Literals); state.OutputLiterals(command.Literals); if (command.CopyDistance == DistanceInfo.EndsAfterLiterals) { lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.PreferEnabled)); break; } if (!command.CopyDistance.FindCodes(original.Header.DistanceParameters, state, validDistanceCodes)) { lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.ForceEnabled)); } else { DistanceCode distanceCode; if (command.CopyDistance == DistanceInfo.ExplicitCodeZero) { distanceCode = DistanceCode.Zero; } else { distanceCode = validDistanceCodes.Count > 1 ? parameters.DistanceCodePicker(validDistanceCodes, distanceFreq) : validDistanceCodes[0]; } distanceFreq.Add(distanceCode); distanceCodes.Add(distanceCode); lengthCodes.Add(command.Lengths.MakeCode(ImplicitDistanceCodeZero.Disable)); } } var origLitCtxMap = builder.LiteralCtxMap; if (origLitCtxMap.TreeCount == 1) { Split(builder, Category.Literal, literals, 512, 400.0); builder.UseSameLiteralContextMode(LiteralContextMode.UTF8); builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).RepeatFirstBlockType(true).Build(); } else { var literalContextMap = Enumerable.Range(0, origLitCtxMap.ContextsPerBlockType).Select(index => origLitCtxMap.DetermineTreeID(0, index)).ToArray(); var literalContextMode = builder.LiteralContextModes[0]; var literalBuffer = RingBufferFast <byte> .From(0, 0); Split(builder, Category.Literal, literals, 512, 400.0, new BlockSplitter <Literal> .ContextInfo(literalContextMap, literal => { literalBuffer.Push(literal.Value); return(literalContextMode.DetermineContextID(literalBuffer.Front, literalBuffer.Back)); })); builder.UseSameLiteralContextMode(literalContextMode); builder.LiteralCtxMap = new ContextMapBuilder.Literals(builder).Set(0, literalContextMap).RepeatFirstBlockType(true).Build(); } Split(builder, Category.InsertCopy, lengthCodes, 1024, 500.0); Split(builder, Category.Distance, distanceCodes, 512, 100.0); builder.DistanceCtxMap = new ContextMapBuilder.Distances(builder).RepeatFirstBlockType(true).Build(); return(builder.Build(parameters)); }
/// <summary> /// Adapted from https://github.com/google/brotli/blob/master/c/enc/backward_references.c (BrotliCreateBackwardReferences). /// </summary> private void CreateBackwardReferences(CompressedMetaBlockBuilder builder, int chunkLength) { int maxBackwardLimit = fileParameters.WindowSize.Bytes; int insertLength = lastInsertLen; int posEnd = position + chunkLength; int storeEnd = chunkLength >= hasher.StoreLookahead ? posEnd - hasher.StoreLookahead + 1 : position; int applyRandomHeuristics = position + LiteralSpreeLengthForSparseSearch; while (position + hasher.HashTypeLength < posEnd) { int maxLength = posEnd - position; int maxDistance = Math.Min(position, maxBackwardLimit); int dictionaryStart = Math.Min(position, maxBackwardLimit); var result = hasher.FindLongestMatch(position, maxLength, maxDistance, dictionaryStart, builder.LastDistance, 0); if (result.FoundAnything) { int delayedBackwardReferencesInRow = 0; --maxLength; while (true) { maxDistance = Math.Min(position + 1, maxBackwardLimit); dictionaryStart = Math.Min(position + 1, maxBackwardLimit); var bestLenIn = features.HasFlag(Features.ExtensiveReferenceSearch) ? 0 : Math.Min(result.Copy.OutputLength - 1, maxLength); var result2 = hasher.FindLongestMatch(position + 1, maxLength, maxDistance, dictionaryStart, builder.LastDistance, bestLenIn); if (result2.Score >= result.Score + CostDiffLazy) { ++position; ++insertLength; result = result2; if (++delayedBackwardReferencesInRow < 4 && position + hasher.HashTypeLength < posEnd) { --maxLength; continue; } } break; } var copy = result.Copy; int len = copy.OutputLength; applyRandomHeuristics = position + (2 * len) + LiteralSpreeLengthForSparseSearch; copy.AddCommand(builder, Literal.FromBytes(input, position - insertLength, insertLength)); insertLength = 0; int rangeStart = position + 2; int rangeEnd = Math.Min(position + len, storeEnd); if (copy is Copy.BackReference backReference && backReference.Distance < len >> 2) { rangeStart = Math.Min(rangeEnd, Math.Max(rangeStart, position + len - (backReference.Distance << 2))); } hasher.StoreRange(rangeStart, rangeEnd); position += len; } else { ++insertLength; ++position; if (position > applyRandomHeuristics) { int skipStep; int nextStopOffset; if (position > applyRandomHeuristics + (4 * LiteralSpreeLengthForSparseSearch)) { skipStep = 4; nextStopOffset = 16; } else { skipStep = 2; nextStopOffset = 8; } int margin = Math.Max(hasher.StoreLookahead - 1, skipStep); int posJump = Math.Min(position + nextStopOffset, posEnd - margin); while (position < posJump) { hasher.Store(position); insertLength += skipStep; position += skipStep; } } } } insertLength += posEnd - position; lastInsertLen = insertLength; }
private NextStep Trawl(CompressedMetaBlockBuilder builder) { do { uint hash = nextHash; int bytesBetweenHashLookups = (skip++) >> 5; ip = nextIp; nextIp = ip + bytesBetweenHashLookups; if (nextIp > ipLimit) { return(NextStep.EmitRemainder); } nextHash = table.Hash(nextIp); candidate = ip - lastDistance; if (Match.Check(input, ip, candidate, MinMatchLen) && candidate < ip) { table[hash] = ip - baseIp; break; } candidate = baseIp + table[hash]; table[hash] = ip - baseIp; }while(!Match.Check(input, ip, candidate, MinMatchLen)); if (ip - candidate > SupportedWindowSize.Bytes) { return(NextStep.Trawl); } { int @base = ip; int matched = MinMatchLen + Match.DetermineLength(input, candidate + MinMatchLen, ip + MinMatchLen, ipEnd - ip - MinMatchLen); int distance = @base - candidate; int insert = @base - nextEmit; ip += matched; if (insert >= 6210 && ShouldUseUncompressedMode(insert)) { outputTo = @base; return(NextStep.OutputUncompressed); } if (distance == lastDistance) { builder.AddInsertCopy(Literal.FromBytes(input, nextEmit, insert), 2, DistanceInfo.ExplicitCodeZero); } else { builder.AddInsertCopy(Literal.FromBytes(input, nextEmit, insert), 2, distance); lastDistance = distance; } builder.AddCopy(matched - 2, distance); nextEmit = ip; if (ip >= ipLimit) { return(NextStep.EmitRemainder); } candidate = table.UpdateAndGetCandidate(ip, baseIp); } while (Match.Check(input, ip, candidate, MinMatchLen)) { int @base = ip; int matched = MinMatchLen + Match.DetermineLength(input, candidate + MinMatchLen, ip + MinMatchLen, ipEnd - ip - MinMatchLen); if (ip - candidate > SupportedWindowSize.Bytes) { break; } ip += matched; lastDistance = @base - candidate; builder.AddCopy(matched, lastDistance); nextEmit = ip; if (ip >= ipLimit) { return(NextStep.EmitRemainder); } candidate = table.UpdateAndGetCandidate(ip, baseIp); } nextHash = table.Hash(++ip); return(NextStep.EmitCommandsNextHash); }
private void TestBlockSplits(CompressedMetaBlockBuilder builder, BrotliCompressionParameters parameters, MetaBlockSizeTracker tracker, Category category) { int totalBlockLength = builder.GetTotalBlockLength(category); if (totalBlockLength == 0) { return; } int step = Math.Max(1, (int)Math.Floor(Math.Log(totalBlockLength, 1.15))); int stepTwice = step * 2; if (totalBlockLength < stepTwice) { return; } var lengths = new List <int> { totalBlockLength }; var queue = new Queue <int>(); queue.Enqueue(0); while (queue.TryDequeue(out int leftIndex)) { if (lengths[leftIndex] < stepTwice) { continue; } int rightIndex = leftIndex + 1; lengths.Insert(rightIndex, lengths[leftIndex] - step); lengths[leftIndex] = step; ApplyBlockSplit(builder.BlockTypes[category], lengths); PrepareContextMap(builder, category, lengths.Count); tracker.Test(builder, parameters); while (lengths[leftIndex] + step < lengths[rightIndex] && lengths[rightIndex] >= stepTwice) { lengths[leftIndex] += step; lengths[rightIndex] -= step; ApplyBlockSplit(builder.BlockTypes[category], lengths); PrepareContextMap(builder, category, lengths.Count); tracker.Test(builder, parameters); } var smallest = tracker.Smallest; if (smallest == null) { return; } var mb = smallest.Value.Item1; int prev = lengths.Count; lengths.Clear(); lengths.Add(mb.Header.BlockTypes[category].InitialLength); lengths.AddRange(mb.Data.BlockSwitchCommands[category].Select(command => command.Length)); var finalLength = totalBlockLength - lengths.Sum(); if (finalLength > 0) { lengths.Add(totalBlockLength - lengths.Sum()); } if (lengths.Count >= prev) { queue.Enqueue(leftIndex); queue.Enqueue(rightIndex); } } }
internal override int AddCommand(CompressedMetaBlockBuilder builder, IList <Literal> literals) { builder.AddInsertCopy(literals, in entry); return(entry.OutputLength); }
internal abstract int AddCommand(CompressedMetaBlockBuilder builder, IList <Literal> literals);