/// <summary>
/// Writes Number + (Sign) + Exponent + Sign
/// </summary>
/// <param name="writer"></param>
/// <param name="value"></param>
protected void WriteSingleValueWithoutControlBit(StreamBitWriter writer, OfcNumber value)
{
if (!Context.Metadata.IsAbsolute)
{
writer.WriteByte(value.IsNegative ? (byte)1 : (byte)0, 1);
}
writer.Write((ulong)Math.Abs(value.Number), Context.Metadata.MaxNeededBitsNumber);
writer.WriteByte(value.Exponent < 0 ? (byte)1 : (byte)0, 1); // Bug: potentually writing exponent even though NoExp is set to true
writer.Write((ushort)Math.Abs(value.Exponent), Context.Metadata.MaxNeededBitsExponent);
}
/// <summary>
/// Writes the metadata in binary form
/// </summary>
/// <param name="writer"></param>
public void Write(StreamBitWriter writer)
{
writer.Write((ulong)ValueCount, 31);
writer.WriteByte(IsAbsolute ? (byte)1 : (byte)0, 1);
if (IsAbsolute)
{
writer.WriteByte(IsNegative ? (byte)1 : (byte)0, 1);
}
writer.WriteByte(MaxNeededBitsExponent, 4);
writer.WriteByte(MaxNeededBitsNumber, 6); // The MaxNeededBitsNeededBitsNumber and LargestPossibleNumber can be calculated from that ...
}
protected void WriteDefaultBlockHeader(StreamBitWriter writer, Block block)
{
writer.WriteByte(1, 1); // IsBlock
if (block.HasExponent)
{
writer.WriteByte(1, 1);
writer.WriteByte((byte)(block.Exponent < 0 ? 1 : 0), 1);
writer.Write((ushort)Math.Abs(block.Exponent), Context.Metadata.MaxNeededBitsExponent);
}
else
{
writer.WriteByte(0, 1);
}
if (block.HasPattern)
{
writer.WriteByte(1, 1);
writer.WriteByte((byte)block.Pattern, 2);
}
else
{
writer.WriteByte(0, 1);
}
writer.WriteByte(block.Length, 8);
}
/// <summary>
/// Writes a value to the stored bit-writer. Will write the value as huffman representation if possible. Otherwise, the value is written as normal value
/// </summary>
/// <param name="num"></param>
public void Write(ushort num)
{
var occurencePlace = _indexMapper[num];
if (occurencePlace == 0)
{
_writer.WriteByte(0, 1); // isHuffman flag
_writer.Write(num, _defaultBitCount); // Writing the normal value, NO huffman representation
}
else
{
_writer.WriteByte(1, 1); // isHuffman flag
_writer.Write(OccurenceNum[occurencePlace], occurencePlace); // Writing huffman representation
}
}
public override void Write(StreamBitWriter writer, Block block, ref int valueIndex)
{
WriteDefaultBlockHeader(writer, block);
if (_probabilityMetadataIndex > PingPongPatternLengths.Count)
{
_probabilityMetadataIndex = 0;
}
var meta = PingPongPatternLengths[_probabilityMetadataIndex++];
if (meta.BlockIndex != block.Index)
{
for (var i = 0; i < PingPongPatternLengths.Count; i++)
{
if (PingPongPatternLengths[i].BlockIndex == block.Index)
{
meta = PingPongPatternLengths[i];
_probabilityMetadataIndex = i++;
break;
}
}
}
WriteSingleValueWithoutControlBit(writer, Values[block.Index]);
WriteSingleValueWithoutControlBit(writer, Values[block.Index + meta.Length]);
writer.WriteByte(meta.Length, 8);
valueIndex += meta.Length * block.Length;
}
public override void Write(StreamBitWriter writer, Block block, ref int valueIndex) //Todo: this method is redundant: same as NoExp. Need more common methods
{
WriteDefaultBlockHeader(writer, block);
if (block.OverrideGlobalNb)
{
writer.WriteByte(1, 1);
writer.Write(Context.Metadata.MaxNeededBitsNumber, Context.Metadata.MaxNeededBitsNeededBitsNumber);
}
else
{
writer.WriteByte(0, 1);
}
if (!Context.Metadata.IsAbsolute)
{
if (block.AbsoluteSign)
{
writer.WriteByte(1, 1);
writer.WriteByte(block.IsSignNegative ? (byte)1 : (byte)0, 1);
}
else
{
writer.WriteByte(0, 1);
}
}
var end = block.Index + block.Length;
for (var i = block.Index; i < end; i++)
{
var value = Values[i];
if (!block.AbsoluteSign)
{
writer.WriteByte(value.IsNegative ? (byte)1 : (byte)0, 1);
}
if (value.Exponent > block.Exponent)
{
writer.Write((ulong)(Math.Pow(10, value.Exponent - block.Exponent) * value.Number), Context.Metadata.MaxNeededBitsNumber);
}
else
{
writer.Write((ulong)value.Number, Context.Metadata.MaxNeededBitsNumber);
}
}
valueIndex += block.Length;
}
private void FinishBlock()
{
var maxNumberLength = 1;
var maxTotalLength = 1;
var numberLengths = new int[_valueIndex];
var expLengths = new int[_valueIndex];
var averageNumberLength = 0;
var averageExpLength = 0;
#region Metadata analysis
for (var i = 0; i < _valueIndex; i++)
{
var currentLength = _values[i].Length;
if (currentLength > maxTotalLength)
{
maxTotalLength = currentLength;
}
numberLengths[i] = currentLength;
for (var j = 0; j < _values[i].Length; j++)
{
if (_values[i][j] == 'e')
{
var expLen = _values[i].Length - (j + 1);
expLengths[i] = expLen;
numberLengths[i] = j;
currentLength = j;
break;
}
}
averageNumberLength += numberLengths[i];
averageExpLength += expLengths[i];
if (currentLength > maxNumberLength)
{
maxNumberLength = currentLength;
}
}
averageNumberLength /= _valueIndex;
averageExpLength /= _valueIndex;
#endregion
_bitWriter.Write((ulong)_valueIndex, 32); // block size
_bitWriter.WriteByte((byte)averageNumberLength, 8);
_bitWriter.WriteByte((byte)averageExpLength, 8);
for (var i = 0; i < _valueIndex; i++)
{
var diff = numberLengths[i] - averageNumberLength;
_bitWriter.WriteByte((byte)(diff < 0 ? 1 : 0), 1);
_bitWriter.Write((ulong)(1 << Math.Abs(diff)), (byte)(Math.Abs(diff) + 1)); // Writing NUM ending masks 001 -> ending in 4 chars
}
for (var i = 0; i < _valueIndex; i++)
{
var diff = expLengths[i] - averageExpLength;
_bitWriter.WriteByte((byte)(diff < 0 ? 1 : 0), 1);
_bitWriter.Write((ulong)(1 << Math.Abs(diff)), (byte)(Math.Abs(diff) + 1)); // Writing NUM ending masks 001 -> ending in 4 chars
}
_bitWriter.Flush();
for (var i = 0; i < maxNumberLength; i++) // writing numbers in 111000 notation
{
for (var j = 0; j < _valueIndex; j++)
{
var str = _values[j];
if (numberLengths[j] > i)
{
_bitWriter.Stream.WriteByte((byte)(str[i] /* - 48*/));
}
}
}
var changed = true;
for (var offset = 0; changed; offset++) // writing exponents in 111000 notation
{
changed = false;
for (var i = 0; i < _valueIndex; i++)
{
var index = numberLengths[i] + 1 + offset;
if (_values[i].Length > index)
{
_bitWriter.Stream.WriteByte((byte)(_values[i][index] /* - 48*/));
changed = true;
}
}
}
_valueIndex = 0;
}
/// <summary>
/// Will write everything to the given Stream/Writer. Call "Compress" first!
/// </summary>
public void Write() //Bug: Actually, finding all blocks and then writing all blocks is pretty stupid. By doing this we first have a CPU / RAM bottleneck, and here we get a Harddrive bottleneck. Doesn't matter if master weed does good parallel stuff though
{
var currentBlockIndex = 0;
var valueCount = Values.Count;
var blockCount = Blocks.Count;
var nextStop = blockCount == 0 ? valueCount : Blocks[currentBlockIndex].Index;
var hasExponent = !Metadata.NoExponent;
var huffmanWriter = _huffman?.CreateWriter(Metadata.MaxNeededBitsExponent, _writer);
var shouldUseHuffman = hasExponent && huffmanWriter != null && huffmanWriter.IsBetterThanDefault(valueCount - _totalPostCompressionOptimisationBlockValues, _huffman);
// Writing global header
Metadata.Write(_writer);
shouldUseHuffman = false; //debug
_writer.WriteByte(shouldUseHuffman ? (byte)1 : (byte)0, 1);
if (shouldUseHuffman)
{
huffmanWriter.WriteDictionary();
}
for (var i = 0; i < valueCount;)
{
for (; nextStop > i; i++) // writing values until the next block is here
{
var value = Values[i];
//Todo: is the loop below really nessecary with the new changes?
while (value.NeededBitsNumber > Metadata.MaxNeededBitsNumber) //Todo: check how often this case appears. (This is a "bug" created by the blockfinding, which "corrects" the value to fit with the exp of a block that might get created ...
{
value.Number /= 10;
value.Exponent++;
value.NeededBitsNumber = value.Number.GetNeededBits();
}
_writer.WriteByte(0, 1);
if (!Metadata.IsAbsolute)
{
_writer.WriteByte((byte)(value.IsNegative ? 1 : 0), 1);
}
_writer.Write((ulong)value.Number, Metadata.MaxNeededBitsNumber);
if (hasExponent)
{
_writer.WriteByte((byte)(value.Exponent < 0 ? 1 : 0), 1);
if (shouldUseHuffman)
{
huffmanWriter.Write((ushort)Math.Abs(value.Exponent));
}
else
{
_writer.Write((ulong)Math.Abs(value.Exponent), Metadata.MaxNeededBitsExponent);
}
}
}
if (++currentBlockIndex < blockCount)
{
// Console.WriteLine(Blocks[currentBlockIndex - 1]);
Blocks[currentBlockIndex - 1].Method.Write(_writer, Blocks[currentBlockIndex - 1], ref i);
nextStop = Blocks[currentBlockIndex].Index;
}
else
{
nextStop = valueCount;
}
}
_writer.Flush(); // This is nessecary. Will write the last buffered byte may only be partially complete!
}
