private static int DecodeContextMap(int contextMapSize, byte[] contextMap, BitReader br) { BitReader.ReadMoreInput(br); int numTrees = DecodeVarLenUnsignedByte(br) + 1; if (numTrees == 1) { Utils.FillWithZeroes(contextMap, 0, contextMapSize); return(numTrees); } bool useRleForZeros = BitReader.ReadBits(br, 1) == 1; int maxRunLengthPrefix = 0; if (useRleForZeros) { maxRunLengthPrefix = BitReader.ReadBits(br, 4) + 1; } int[] table = new int[Huffman.HUFFMAN_MAX_TABLE_SIZE]; ReadHuffmanCode(numTrees + maxRunLengthPrefix, table, 0, br); for (int i = 0; i < contextMapSize;) { BitReader.ReadMoreInput(br); int code = ReadSymbol(table, 0, br); if (code == 0) { contextMap[i] = 0; i++; } else if (code <= maxRunLengthPrefix) { int reps = (1 << code) + BitReader.ReadBits(br, code); while (reps != 0) { if (i >= contextMapSize) { throw new BrotliRuntimeException("Corrupted context map"); // COV_NF_LINE } contextMap[i] = 0; i++; reps--; } } else { contextMap[i] = (byte)(code - maxRunLengthPrefix); i++; } } if (BitReader.ReadBits(br, 1) == 1) { InverseMoveToFrontTransform(contextMap, contextMapSize); } return(numTrees); }
/// <summary> /// Reads next metablock header. /// </summary> /// <param name="state">decoding state</param> public static void ReadMeablockInfo(State state) { BitReader br = state.br; if (state.inputEnd) { state.nextRunningState = RunningStage.FINISHED; state.bytesToWrite = state.pos & (state.ringBufferSize - 1); state.bytesWritten = 0; state.runningState = RunningStage.WRITE; return; } // TODO: Reset? Do we need this? state.hGroup0.codes = null; state.hGroup0.trees = null; state.hGroup1.codes = null; state.hGroup1.trees = null; state.hGroup2.codes = null; state.hGroup2.trees = null; BitReader.ReadMoreInput(br); DecodeMetaBlockLength(br, state); if (state.metaBlockLength == 0 && !state.isMetadata) { return; } if (state.isUncompressed || state.isMetadata) { BitReader.JumpToByteBoundry(br); state.runningState = state.isMetadata ? RunningStage.READ_METADATA : RunningStage.COPY_UNCOMPRESSED; } else { state.runningState = RunningStage.COMPRESSED_BLOCK_START; } if (state.isMetadata) { return; } state.expectedTotalSize += state.metaBlockLength; if (state.ringBufferSize < state.maxRingBufferSize) { MaybeReallocateRingBuffer(state); } }
public static void CopyUncompressedData(State state) { BitReader br = state.br; byte[] ringBuffer = state.ringBuffer; int ringBufferMask = state.ringBufferSize - 1; while (state.metaBlockLength > 0) { BitReader.ReadMoreInput(br); // Optimize ringBuffer[state.pos & ringBufferMask] = (byte)(BitReader.ReadBits(br, 8)); state.metaBlockLength--; if ((state.pos++ & ringBufferMask) == ringBufferMask) { state.nextRunningState = RunningStage.COPY_UNCOMPRESSED; state.bytesToWrite = state.ringBufferSize; state.bytesWritten = 0; state.runningState = RunningStage.WRITE; return; } } state.runningState = RunningStage.BLOCK_START; }
public static void Decompress(State state) { if (state.runningState == RunningStage.UNINITIALIZED) { throw new InvalidOperationException("Can't decompress until initialized"); } if (state.runningState == RunningStage.CLOSED) { throw new InvalidOperationException("Can't decompress after close"); } BitReader br = state.br; int ringBufferMask = state.ringBufferSize - 1; byte[] ringBuffer = state.ringBuffer; while (state.runningState != RunningStage.FINISHED) { // TODO: extract cases to methods for the better readability. switch (state.runningState) { case RunningStage.BLOCK_START: if (state.metaBlockLength < 0) { throw new BrotliRuntimeException("Invalid metablock length"); } ReadMeablockInfo(state); /* Ring-buffer would be reallocated here. */ ringBufferMask = state.ringBufferSize - 1; ringBuffer = state.ringBuffer; continue; case RunningStage.COMPRESSED_BLOCK_START: ReadMetablockHuffmanCodesAndContextMaps(state); state.runningState = RunningStage.MAIN_LOOP; goto case RunningStage.MAIN_LOOP; // Fall through case RunningStage.MAIN_LOOP: if (state.metaBlockLength <= 0) { // Protect pos from overflow, wrap it around at every GB of input data. state.pos &= 0x3fffffff; state.runningState = RunningStage.BLOCK_START; continue; } BitReader.ReadMoreInput(br); if (state.blockLength[1] == 0) { DecodeCommandBlockSwitch(state); } state.blockLength[1]--; int cmdCode = ReadSymbol(state.hGroup1.codes, state.treeCommandOffset, br); int rangeIdx = (int)((uint)cmdCode >> 6); state.distanceCode = 0; if (rangeIdx >= 2) { rangeIdx -= 2; state.distanceCode = -1; } int insertCode = Prefix.INSERT_RANGE_LUT[rangeIdx] + ((int)((uint)cmdCode >> 3) & 7); int copyCode = Prefix.COPY_RANGE_LUT[rangeIdx] + (cmdCode & 7); state.insertLength = Prefix.INSERT_LENGTH_OFFSET[insertCode] + BitReader .ReadBits(br, Prefix.INSERT_LENGTH_N_BITS[insertCode]); state.copyLength = Prefix.COPY_LENGTH_OFFSET[copyCode] + BitReader .ReadBits(br, Prefix.COPY_LENGTH_N_BITS[copyCode]); state.j = 0; state.runningState = RunningStage.INSERT_LOOP; goto case RunningStage.INSERT_LOOP; // Fall through case RunningStage.INSERT_LOOP: if (state.trivialLiteralContext) { while (state.j < state.insertLength) { BitReader.ReadMoreInput(br); if (state.blockLength[0] == 0) { DecodeLiteralBlockSwitch(state); } state.blockLength[0]--; ringBuffer[state.pos & ringBufferMask] = (byte)ReadSymbol( state.hGroup0.codes, state.literalTree, br); state.j++; if ((state.pos++ & ringBufferMask) == ringBufferMask) { state.nextRunningState = RunningStage.INSERT_LOOP; state.bytesToWrite = state.ringBufferSize; state.bytesWritten = 0; state.runningState = RunningStage.WRITE; break; } } } else { int prevByte1 = ringBuffer[(state.pos - 1) & ringBufferMask] & 0xFF; int prevByte2 = ringBuffer[(state.pos - 2) & ringBufferMask] & 0xFF; while (state.j < state.insertLength) { BitReader.ReadMoreInput(br); if (state.blockLength[0] == 0) { DecodeLiteralBlockSwitch(state); } int literalTreeIndex = state.contextMap[state.contextMapSlice + (Context.LOOKUP[state.contextLookupOffset1 + prevByte1] | Context.LOOKUP[state.contextLookupOffset2 + prevByte2])] & 0xFF; state.blockLength[0]--; prevByte2 = prevByte1; prevByte1 = ReadSymbol( state.hGroup0.codes, state.hGroup0.trees[literalTreeIndex], br); ringBuffer[state.pos & ringBufferMask] = (byte)prevByte1; state.j++; if ((state.pos++ & ringBufferMask) == ringBufferMask) { state.nextRunningState = RunningStage.INSERT_LOOP; state.bytesToWrite = state.ringBufferSize; state.bytesWritten = 0; state.runningState = RunningStage.WRITE; break; } } } if (state.runningState != RunningStage.INSERT_LOOP) { continue; } state.metaBlockLength -= state.insertLength; if (state.metaBlockLength <= 0) { state.runningState = RunningStage.MAIN_LOOP; continue; } if (state.distanceCode < 0) { BitReader.ReadMoreInput(br); if (state.blockLength[2] == 0) { DecodeDistanceBlockSwitch(state); } state.blockLength[2]--; state.distanceCode = ReadSymbol(state.hGroup2.codes, state.hGroup2.trees[ state.distContextMap[state.distContextMapSlice + (state.copyLength > 4 ? 3 : state.copyLength - 2)] & 0xFF], br); if (state.distanceCode >= state.numDirectDistanceCodes) { state.distanceCode -= state.numDirectDistanceCodes; int postfix = state.distanceCode & state.distancePostfixMask; //state.distanceCode >>>= state.distancePostfixBits; state.distanceCode = (int)((uint)state.distanceCode >> state.distancePostfixBits); int n = (int)((uint)state.distanceCode >> 1) + 1; int offset = ((2 + (state.distanceCode & 1)) << n) - 4; state.distanceCode = state.numDirectDistanceCodes + postfix + ((offset + BitReader.ReadBits(br, n)) << state.distancePostfixBits); } } // Convert the distance code to the actual distance by possibly looking up past distances // from the ringBuffer. state.distance = TranslateShortCodes(state.distanceCode, state.distRb, state.distRbIdx); if (state.distance < 0) { throw new BrotliRuntimeException("Negative distance"); // COV_NF_LINE } if (state.pos < state.maxBackwardDistance && state.maxDistance != state.maxBackwardDistance) { state.maxDistance = state.pos; } else { state.maxDistance = state.maxBackwardDistance; } state.copyDst = state.pos & ringBufferMask; if (state.distance > state.maxDistance) { state.runningState = RunningStage.TRANSFORM; continue; } if (state.distanceCode > 0) { state.distRb[state.distRbIdx & 3] = state.distance; state.distRbIdx++; } if (state.copyLength > state.metaBlockLength) { throw new BrotliRuntimeException("Invalid backward reference"); // COV_NF_LINE } state.j = 0; state.runningState = RunningStage.COPY_LOOP; goto case RunningStage.COPY_LOOP; // fall through case RunningStage.COPY_LOOP: for (; state.j < state.copyLength;) { ringBuffer[state.pos & ringBufferMask] = ringBuffer[(state.pos - state.distance) & ringBufferMask]; // TODO: condense state.metaBlockLength--; state.j++; if ((state.pos++ & ringBufferMask) == ringBufferMask) { state.nextRunningState = RunningStage.COPY_LOOP; state.bytesToWrite = state.ringBufferSize; state.bytesWritten = 0; state.runningState = RunningStage.WRITE; break; } } if (state.runningState == RunningStage.COPY_LOOP) { state.runningState = RunningStage.MAIN_LOOP; } continue; case RunningStage.TRANSFORM: if (state.copyLength >= Dictionary.MIN_WORD_LENGTH && state.copyLength <= Dictionary.MAX_WORD_LENGTH) { int offset = Dictionary.OFFSETS_BY_LENGTH[state.copyLength]; int wordId = state.distance - state.maxDistance - 1; int shift = Dictionary.SIZE_BITS_BY_LENGTH[state.copyLength]; int mask = (1 << shift) - 1; int wordIdx = wordId & mask; int transformIdx = (int)((uint)wordId >> shift); offset += wordIdx * state.copyLength; if (transformIdx < Transform.TRANSFORMS.Length) { int len = Transform.TransformDictionaryWord(ringBuffer, state.copyDst, Dictionary.GetData(), offset, state.copyLength, Transform.TRANSFORMS[transformIdx]); state.copyDst += len; state.pos += len; state.metaBlockLength -= len; if (state.copyDst >= state.ringBufferSize) { state.nextRunningState = RunningStage.COPY_WRAP_BUFFER; state.bytesToWrite = state.ringBufferSize; state.bytesWritten = 0; state.runningState = RunningStage.WRITE; continue; } } else { throw new BrotliRuntimeException("Invalid backward reference"); // COV_NF_LINE } } else { throw new BrotliRuntimeException("Invalid backward reference"); // COV_NF_LINE } state.runningState = RunningStage.MAIN_LOOP; continue; case RunningStage.COPY_WRAP_BUFFER: Array.Copy(ringBuffer, state.ringBufferSize, ringBuffer, 0, state.copyDst - state.ringBufferSize); state.runningState = RunningStage.MAIN_LOOP; continue; case RunningStage.READ_METADATA: while (state.metaBlockLength > 0) { BitReader.ReadMoreInput(br); // Optimize BitReader.ReadBits(br, 8); state.metaBlockLength--; } state.runningState = RunningStage.BLOCK_START; continue; case RunningStage.COPY_UNCOMPRESSED: CopyUncompressedData(state); continue; case RunningStage.WRITE: if (!WriteRingBuffer(state)) { // Output buffer is full. return; } state.runningState = state.nextRunningState; continue; default: throw new BrotliRuntimeException("Unexpected state " + state.runningState); } } if (state.runningState == RunningStage.FINISHED) { if (state.metaBlockLength < 0) { throw new BrotliRuntimeException("Invalid metablock length"); } BitReader.JumpToByteBoundry(br); BitReader.CheckHealth(state.br); } }
public static void ReadMetablockHuffmanCodesAndContextMaps(State state) { BitReader br = state.br; for (int i = 0; i < 3; i++) { state.numBlockTypes[i] = DecodeVarLenUnsignedByte(br) + 1; state.blockLength[i] = 1 << 28; if (state.numBlockTypes[i] > 1) { ReadHuffmanCode(state.numBlockTypes[i] + 2, state.blockTypeTrees, i * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); ReadHuffmanCode(NUM_BLOCK_LENGTH_CODES, state.blockLenTrees, i * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); state.blockLength[i] = ReadBlockLength(state.blockLenTrees, i * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); } } BitReader.ReadMoreInput(br); state.distancePostfixBits = BitReader.ReadBits(br, 2); state.numDirectDistanceCodes = NUM_DISTANCE_SHORT_CODES + (BitReader.ReadBits(br, 4) << state.distancePostfixBits); state.distancePostfixMask = (1 << state.distancePostfixBits) - 1; int numDistanceCodes = state.numDirectDistanceCodes + (48 << state.distancePostfixBits); // TODO: Reuse? state.contextModes = new byte[state.numBlockTypes[0]]; for (int i = 0; i < state.numBlockTypes[0];) { /* Ensure that less than 256 bits read between readMoreInput. */ int limit = Math.Min(i + 96, state.numBlockTypes[0]); for (; i < limit; ++i) { state.contextModes[i] = (byte)(BitReader.ReadBits(br, 2) << 1); } BitReader.ReadMoreInput(br); } // TODO: Reuse? state.contextMap = new byte[state.numBlockTypes[0] << LITERAL_CONTEXT_BITS]; int numLiteralTrees = DecodeContextMap(state.numBlockTypes[0] << LITERAL_CONTEXT_BITS, state.contextMap, br); state.trivialLiteralContext = true; for (int j = 0; j < state.numBlockTypes[0] << LITERAL_CONTEXT_BITS; j++) { if (state.contextMap[j] != j >> LITERAL_CONTEXT_BITS) { state.trivialLiteralContext = false; break; } } // TODO: Reuse? state.distContextMap = new byte[state.numBlockTypes[2] << DISTANCE_CONTEXT_BITS]; int numDistTrees = DecodeContextMap(state.numBlockTypes[2] << DISTANCE_CONTEXT_BITS, state.distContextMap, br); HuffmanTreeGroup.Init(state.hGroup0, NUM_LITERAL_CODES, numLiteralTrees); HuffmanTreeGroup.Init(state.hGroup1, NUM_INSERT_AND_COPY_CODES, state.numBlockTypes[1]); HuffmanTreeGroup.Init(state.hGroup2, numDistanceCodes, numDistTrees); HuffmanTreeGroup.Decode(state.hGroup0, br); HuffmanTreeGroup.Decode(state.hGroup1, br); HuffmanTreeGroup.Decode(state.hGroup2, br); state.contextMapSlice = 0; state.distContextMapSlice = 0; state.contextLookupOffset1 = Context.LOOKUP_OFFSETS[state.contextModes[0]]; state.contextLookupOffset2 = Context.LOOKUP_OFFSETS[state.contextModes[0] + 1]; state.literalTreeIndex = 0; state.literalTree = state.hGroup0.trees[0]; state.treeCommandOffset = state.hGroup1.trees[0]; // TODO: == 0? state.blockTypeRb[0] = state.blockTypeRb[2] = state.blockTypeRb[4] = 1; state.blockTypeRb[1] = state.blockTypeRb[3] = state.blockTypeRb[5] = 0; }
public static void ReadHuffmanCode(int alphabetSize, int[] table, int offset, BitReader br) { bool ok = true; int simpleCodeOrSkip; BitReader.ReadMoreInput(br); // TODO: Avoid allocation. int[] codeLengths = new int[alphabetSize]; simpleCodeOrSkip = BitReader.ReadBits(br, 2); if (simpleCodeOrSkip == 1) { // Read symbols, codes & code lengths directly. int maxBitsCounter = alphabetSize - 1; int maxBits = 0; int[] symbols = new int[4]; int numSymbols = BitReader.ReadBits(br, 2) + 1; while (maxBitsCounter != 0) { maxBitsCounter >>= 1; maxBits++; } Utils.FillWithZeroes(codeLengths, 0, alphabetSize); for (int i = 0; i < numSymbols; i++) { symbols[i] = BitReader.ReadBits(br, maxBits) % alphabetSize; codeLengths[symbols[i]] = 2; } codeLengths[symbols[0]] = 1; switch (numSymbols) { case 1: break; case 2: ok = symbols[0] != symbols[1]; codeLengths[symbols[1]] = 1; break; case 3: ok = symbols[0] != symbols[1] && symbols[0] != symbols[2] && symbols[1] != symbols[2]; break; case 4: ok = symbols[0] != symbols[1] && symbols[0] != symbols[2] && symbols[0] != symbols[3] && symbols[1] != symbols[2] && symbols[1] != symbols[3] && symbols[2] != symbols[3]; if (BitReader.ReadBits(br, 1) == 1) { codeLengths[symbols[2]] = 3; codeLengths[symbols[3]] = 3; } else { codeLengths[symbols[0]] = 2; } break; } } else { // Decode Huffman-coded code lengths. int[] codeLengthCodeLengths = new int[CODE_LENGTH_CODES]; int space = 32; int numCodes = 0; for (int i = simpleCodeOrSkip; i < CODE_LENGTH_CODES && space > 0; i++) { int codeLenIdx = CODE_LENGTH_CODE_ORDER[i]; BitReader.FillBitWindow(br); int p = (int)((ulong)br.accumulator >> br.bitOffset) & 15; // TODO: Demultiplex FIXED_TABLE. br.bitOffset += FIXED_TABLE[p] >> 16; int v = FIXED_TABLE[p] & 0xFFFF; codeLengthCodeLengths[codeLenIdx] = v; if (v != 0) { space -= (32 >> v); numCodes++; } } ok = (numCodes == 1 || space == 0); ReadHuffmanCodeLengths(codeLengthCodeLengths, alphabetSize, codeLengths, br); } if (!ok) { throw new BrotliRuntimeException("Can't readHuffmanCode"); // COV_NF_LINE } Huffman.BuildHuffmanTable(table, offset, HUFFMAN_TABLE_BITS, codeLengths, alphabetSize); }
private static void ReadHuffmanCodeLengths( int[] codeLengthCodeLengths, int numSymbols, int[] codeLengths, BitReader br) { int symbol = 0; int prevCodeLen = DEFAULT_CODE_LENGTH; int repeat = 0; int repeatCodeLen = 0; int space = 32768; int[] table = new int[32]; Huffman.BuildHuffmanTable(table, 0, 5, codeLengthCodeLengths, CODE_LENGTH_CODES); while (symbol < numSymbols && space > 0) { BitReader.ReadMoreInput(br); BitReader.FillBitWindow(br); int p = (int)(((ulong)br.accumulator >> br.bitOffset)) & 31; br.bitOffset += table[p] >> 16; int codeLen = table[p] & 0xFFFF; if (codeLen < CODE_LENGTH_REPEAT_CODE) { repeat = 0; codeLengths[symbol++] = codeLen; if (codeLen != 0) { prevCodeLen = codeLen; space -= 32768 >> codeLen; } } else { int extraBits = codeLen - 14; int newLen = 0; if (codeLen == CODE_LENGTH_REPEAT_CODE) { newLen = prevCodeLen; } if (repeatCodeLen != newLen) { repeat = 0; repeatCodeLen = newLen; } int oldRepeat = repeat; if (repeat > 0) { repeat -= 2; repeat <<= extraBits; } repeat += BitReader.ReadBits(br, extraBits) + 3; int repeatDelta = repeat - oldRepeat; if (symbol + repeatDelta > numSymbols) { throw new BrotliRuntimeException("symbol + repeatDelta > numSymbols"); // COV_NF_LINE } for (int i = 0; i < repeatDelta; i++) { codeLengths[symbol++] = repeatCodeLen; } if (repeatCodeLen != 0) { space -= repeatDelta << (15 - repeatCodeLen); } } } if (space != 0) { throw new BrotliRuntimeException("Unused space"); // COV_NF_LINE } // TODO: Pass max_symbol to Huffman table builder instead? Utils.FillWithZeroes(codeLengths, symbol, numSymbols - symbol); }