/// <summary> /// Writes Number + (Sign) + Exponent + Sign /// </summary> /// <param name="writer"></param> /// <param name="value"></param> protected void WriteSingleValueWithoutControlBit(StreamBitWriter writer, OfcNumber value) { if (!Context.Metadata.IsAbsolute) { writer.WriteByte(value.IsNegative ? (byte)1 : (byte)0, 1); } writer.Write((ulong)Math.Abs(value.Number), Context.Metadata.MaxNeededBitsNumber); writer.WriteByte(value.Exponent < 0 ? (byte)1 : (byte)0, 1); // Bug: potentually writing exponent even though NoExp is set to true writer.Write((ushort)Math.Abs(value.Exponent), Context.Metadata.MaxNeededBitsExponent); }
/// <summary> /// Writes the metadata in binary form /// </summary> /// <param name="writer"></param> public void Write(StreamBitWriter writer) { writer.Write((ulong)ValueCount, 31); writer.WriteByte(IsAbsolute ? (byte)1 : (byte)0, 1); if (IsAbsolute) { writer.WriteByte(IsNegative ? (byte)1 : (byte)0, 1); } writer.WriteByte(MaxNeededBitsExponent, 4); writer.WriteByte(MaxNeededBitsNumber, 6); // The MaxNeededBitsNeededBitsNumber and LargestPossibleNumber can be calculated from that ... }
protected void WriteDefaultBlockHeader(StreamBitWriter writer, Block block) { writer.WriteByte(1, 1); // IsBlock if (block.HasExponent) { writer.WriteByte(1, 1); writer.WriteByte((byte)(block.Exponent < 0 ? 1 : 0), 1); writer.Write((ushort)Math.Abs(block.Exponent), Context.Metadata.MaxNeededBitsExponent); } else { writer.WriteByte(0, 1); } if (block.HasPattern) { writer.WriteByte(1, 1); writer.WriteByte((byte)block.Pattern, 2); } else { writer.WriteByte(0, 1); } writer.WriteByte(block.Length, 8); }
/// <summary> /// Writes a value to the stored bit-writer. Will write the value as huffman representation if possible. Otherwise, the value is written as normal value /// </summary> /// <param name="num"></param> public void Write(ushort num) { var occurencePlace = _indexMapper[num]; if (occurencePlace == 0) { _writer.WriteByte(0, 1); // isHuffman flag _writer.Write(num, _defaultBitCount); // Writing the normal value, NO huffman representation } else { _writer.WriteByte(1, 1); // isHuffman flag _writer.Write(OccurenceNum[occurencePlace], occurencePlace); // Writing huffman representation } }
public override void Write(StreamBitWriter writer, Block block, ref int valueIndex) { WriteDefaultBlockHeader(writer, block); if (_probabilityMetadataIndex > PingPongPatternLengths.Count) { _probabilityMetadataIndex = 0; } var meta = PingPongPatternLengths[_probabilityMetadataIndex++]; if (meta.BlockIndex != block.Index) { for (var i = 0; i < PingPongPatternLengths.Count; i++) { if (PingPongPatternLengths[i].BlockIndex == block.Index) { meta = PingPongPatternLengths[i]; _probabilityMetadataIndex = i++; break; } } } WriteSingleValueWithoutControlBit(writer, Values[block.Index]); WriteSingleValueWithoutControlBit(writer, Values[block.Index + meta.Length]); writer.WriteByte(meta.Length, 8); valueIndex += meta.Length * block.Length; }
public override void Write(StreamBitWriter writer, Block block, ref int valueIndex) //Todo: this method is redundant: same as NoExp. Need more common methods { WriteDefaultBlockHeader(writer, block); if (block.OverrideGlobalNb) { writer.WriteByte(1, 1); writer.Write(Context.Metadata.MaxNeededBitsNumber, Context.Metadata.MaxNeededBitsNeededBitsNumber); } else { writer.WriteByte(0, 1); } if (!Context.Metadata.IsAbsolute) { if (block.AbsoluteSign) { writer.WriteByte(1, 1); writer.WriteByte(block.IsSignNegative ? (byte)1 : (byte)0, 1); } else { writer.WriteByte(0, 1); } } var end = block.Index + block.Length; for (var i = block.Index; i < end; i++) { var value = Values[i]; if (!block.AbsoluteSign) { writer.WriteByte(value.IsNegative ? (byte)1 : (byte)0, 1); } if (value.Exponent > block.Exponent) { writer.Write((ulong)(Math.Pow(10, value.Exponent - block.Exponent) * value.Number), Context.Metadata.MaxNeededBitsNumber); } else { writer.Write((ulong)value.Number, Context.Metadata.MaxNeededBitsNumber); } } valueIndex += block.Length; }
private void FinishBlock() { var maxNumberLength = 1; var maxTotalLength = 1; var numberLengths = new int[_valueIndex]; var expLengths = new int[_valueIndex]; var averageNumberLength = 0; var averageExpLength = 0; #region Metadata analysis for (var i = 0; i < _valueIndex; i++) { var currentLength = _values[i].Length; if (currentLength > maxTotalLength) { maxTotalLength = currentLength; } numberLengths[i] = currentLength; for (var j = 0; j < _values[i].Length; j++) { if (_values[i][j] == 'e') { var expLen = _values[i].Length - (j + 1); expLengths[i] = expLen; numberLengths[i] = j; currentLength = j; break; } } averageNumberLength += numberLengths[i]; averageExpLength += expLengths[i]; if (currentLength > maxNumberLength) { maxNumberLength = currentLength; } } averageNumberLength /= _valueIndex; averageExpLength /= _valueIndex; #endregion _bitWriter.Write((ulong)_valueIndex, 32); // block size _bitWriter.WriteByte((byte)averageNumberLength, 8); _bitWriter.WriteByte((byte)averageExpLength, 8); for (var i = 0; i < _valueIndex; i++) { var diff = numberLengths[i] - averageNumberLength; _bitWriter.WriteByte((byte)(diff < 0 ? 1 : 0), 1); _bitWriter.Write((ulong)(1 << Math.Abs(diff)), (byte)(Math.Abs(diff) + 1)); // Writing NUM ending masks 001 -> ending in 4 chars } for (var i = 0; i < _valueIndex; i++) { var diff = expLengths[i] - averageExpLength; _bitWriter.WriteByte((byte)(diff < 0 ? 1 : 0), 1); _bitWriter.Write((ulong)(1 << Math.Abs(diff)), (byte)(Math.Abs(diff) + 1)); // Writing NUM ending masks 001 -> ending in 4 chars } _bitWriter.Flush(); for (var i = 0; i < maxNumberLength; i++) // writing numbers in 111000 notation { for (var j = 0; j < _valueIndex; j++) { var str = _values[j]; if (numberLengths[j] > i) { _bitWriter.Stream.WriteByte((byte)(str[i] /* - 48*/)); } } } var changed = true; for (var offset = 0; changed; offset++) // writing exponents in 111000 notation { changed = false; for (var i = 0; i < _valueIndex; i++) { var index = numberLengths[i] + 1 + offset; if (_values[i].Length > index) { _bitWriter.Stream.WriteByte((byte)(_values[i][index] /* - 48*/)); changed = true; } } } _valueIndex = 0; }
/// <summary> /// Will write everything to the given Stream/Writer. Call "Compress" first! /// </summary> public void Write() //Bug: Actually, finding all blocks and then writing all blocks is pretty stupid. By doing this we first have a CPU / RAM bottleneck, and here we get a Harddrive bottleneck. Doesn't matter if master weed does good parallel stuff though { var currentBlockIndex = 0; var valueCount = Values.Count; var blockCount = Blocks.Count; var nextStop = blockCount == 0 ? valueCount : Blocks[currentBlockIndex].Index; var hasExponent = !Metadata.NoExponent; var huffmanWriter = _huffman?.CreateWriter(Metadata.MaxNeededBitsExponent, _writer); var shouldUseHuffman = hasExponent && huffmanWriter != null && huffmanWriter.IsBetterThanDefault(valueCount - _totalPostCompressionOptimisationBlockValues, _huffman); // Writing global header Metadata.Write(_writer); shouldUseHuffman = false; //debug _writer.WriteByte(shouldUseHuffman ? (byte)1 : (byte)0, 1); if (shouldUseHuffman) { huffmanWriter.WriteDictionary(); } for (var i = 0; i < valueCount;) { for (; nextStop > i; i++) // writing values until the next block is here { var value = Values[i]; //Todo: is the loop below really nessecary with the new changes? while (value.NeededBitsNumber > Metadata.MaxNeededBitsNumber) //Todo: check how often this case appears. (This is a "bug" created by the blockfinding, which "corrects" the value to fit with the exp of a block that might get created ... { value.Number /= 10; value.Exponent++; value.NeededBitsNumber = value.Number.GetNeededBits(); } _writer.WriteByte(0, 1); if (!Metadata.IsAbsolute) { _writer.WriteByte((byte)(value.IsNegative ? 1 : 0), 1); } _writer.Write((ulong)value.Number, Metadata.MaxNeededBitsNumber); if (hasExponent) { _writer.WriteByte((byte)(value.Exponent < 0 ? 1 : 0), 1); if (shouldUseHuffman) { huffmanWriter.Write((ushort)Math.Abs(value.Exponent)); } else { _writer.Write((ulong)Math.Abs(value.Exponent), Metadata.MaxNeededBitsExponent); } } } if (++currentBlockIndex < blockCount) { // Console.WriteLine(Blocks[currentBlockIndex - 1]); Blocks[currentBlockIndex - 1].Method.Write(_writer, Blocks[currentBlockIndex - 1], ref i); nextStop = Blocks[currentBlockIndex].Index; } else { nextStop = valueCount; } } _writer.Flush(); // This is nessecary. Will write the last buffered byte may only be partially complete! }