public override int Read(IOfcNumberWriter writer, Block block, StreamBitReader reader)
{
// ReSharper disable once AssignmentInConditionalExpression
if (block.OverrideGlobalNb = reader.ReadByte(1) > 0)
{
block.NeededBits = reader.ReadByte(Metadata.MaxNeededBitsNeededBitsNumber);
}
// ReSharper disable once AssignmentInConditionalExpression
if (!Metadata.IsAbsolute)
{
if (block.AbsoluteSign = reader.ReadByte(1) > 0)
{
block.IsSignNegative = reader.ReadByte(1) > 0;
}
}
var hasAbsoluteSign = block.AbsoluteSign || Metadata.IsAbsolute;
var isNegative = block.AbsoluteSign && block.IsSignNegative || Metadata.IsAbsolute && Metadata.IsNegative;
for (var i = 0; i < block.Length; i++)
{
var num = new OfcNumber();
if (!hasAbsoluteSign)
{
isNegative = reader.ReadByte(1) > 0;
}
num.Number = ((long)reader.Read(Metadata.MaxNeededBitsNumber)) * (isNegative ? -1 : 1);
writer.Write(num);
}
return(block.Length);
}
/// <summary>
/// Updates the potentual blocks that could replace the current block
/// </summary>
/// <param name="value"></param>
/// <param name="bitDiffDiff"></param>
private void UpdateReplacingCalculations(ref OfcNumber value, int bitDiffDiff = 0)
{
// ReSharper disable once ForCanBeConvertedToForeach
for (var j = _appendingCalculationSavingGrade + 1; j < _replacingCalculations.Length; j++)
{
if (!_replacingCalculations[j].IsValid)
{
continue;
}
var replacingCalculation = _replacingCalculations[j];
var savingGrade = replacingCalculation.VirtualBlock.GetSavingGrade();
if (!replacingCalculation.ProcessValue(value, _index))
{
_replacingCalculations[j].IsValid = false;
continue;
}
if ((replacingCalculation.SavedBits += bitDiffDiff) > 0)
{
_replacingCalculations[j].IsValid = false;
_calculations.Clear(); //Todo: not jsut clear, but make a replacingCalc on this replacing calc ...
_hasRunningPatternCalculation = false;
if (!replacingCalculation.OldConcurrentBlock.IsValid) //Todo: check what this was
{
ReplaceNewestBlock(BlockCalculation.FromReplacingCalculation(replacingCalculation));
}
else
{
_blocks[_blocks.Count - 1] = replacingCalculation.OldConcurrentBlock; // set last block to old, non-overlapping state
AddNewBlock(BlockCalculation.FromReplacingCalculation(replacingCalculation));
}
break;
}
var newSavingGrade = replacingCalculation.VirtualBlock.GetSavingGrade();
if (savingGrade != newSavingGrade)
{
_replacingCalculations[(int)savingGrade].IsValid = false;
var overriddenCalc = _replacingCalculations[(int)newSavingGrade];
if (overriddenCalc.IsValid && overriddenCalc.SavedBits > replacingCalculation.SavedBits)
{
continue;
}
}
if (newSavingGrade <= replacingCalculation.OldConcurrentSavingGrade)
{
replacingCalculation.IsValid = false;
}
// replacingCalculations[j].IsValid = false;
_replacingCalculations[(int)newSavingGrade] = replacingCalculation;
}
}
public override bool ProcessValue(ref Block block, OfcNumber value, int index, ref int bitDiff) // The block length will always be at least 1 at this point!
{
if (block.Length == byte.MaxValue)
{
return(false);
}
var firstValue = Values[block.Index];
var secondValue = Values[block.Index + 1];
var lastValue = Values[index - 1];
if (firstValue.Number - secondValue.Number != lastValue.Number - value.Number || firstValue.Exponent - secondValue.Exponent != lastValue.Exponent - value.Exponent)
{
if (firstValue.Exponent == 0 && secondValue.Exponent == 0)
{
bitDiff += Context.Headers.StandardBlockPatternOffset - Context.Headers.StandardBlockNumbersNoExp;
block.Method = Context.GetInitializedMethod(Blockfinding.Blockfinding.Methods.NumbersNoExp);
}
else if (firstValue.Exponent == secondValue.Exponent && secondValue.Exponent == value.Exponent)
{
bitDiff += Context.Headers.StandardBlockPatternOffset - Context.Headers.StandardBlockFloatSimmilar;
block.Method = Context.GetInitializedMethod(Blockfinding.Blockfinding.Methods.FloatSimmilar);
block.Exponent = firstValue.Exponent;
}
else
{
return(false); //todo: maybe the floatsimmilar algorithm could adjust that ...
}
//bitDiff -= _singleValueBits * block.Length - block.Length * MaxNeededBitsExponent * 2;
block.HasPattern = false;
block.BiggestNumber = Math.Max(value.Number, Math.Max(firstValue.Number, lastValue.Number));
if (firstValue.IsNegative == lastValue.IsNegative && lastValue.IsNegative == value.IsNegative)
{
block.AbsoluteSign = true;
block.IsSignNegative = value.IsNegative;
//bitDiff--; commented this out, as the isNegative is now in the default header!
}
else
{
bitDiff -= block.Length;
}
var nb = block.BiggestNumber.GetNeededBits();
block.NeededBits = 0; // the nb per value if we still had the pattern
bitDiff += block.DifferenceWithNb(Context.Metadata, ref nb);
block.NeededBits = nb;
block.Length++;
return(true);
}
block.Length++;
bitDiff += Context.Headers.StandardSingleValue;
return(true);
}
/// <summary>
/// Writes Number + (Sign) + Exponent + Sign
/// </summary>
/// <param name="writer"></param>
/// <param name="value"></param>
protected void WriteSingleValueWithoutControlBit(StreamBitWriter writer, OfcNumber value)
{
if (!Context.Metadata.IsAbsolute)
{
writer.WriteByte(value.IsNegative ? (byte)1 : (byte)0, 1);
}
writer.Write((ulong)Math.Abs(value.Number), Context.Metadata.MaxNeededBitsNumber);
writer.WriteByte(value.Exponent < 0 ? (byte)1 : (byte)0, 1); // Bug: potentually writing exponent even though NoExp is set to true
writer.Write((ushort)Math.Abs(value.Exponent), Context.Metadata.MaxNeededBitsExponent);
}
/// <summary>
/// Is able to say if there's could be a pattern in the near future
/// </summary>
/// <param name="current"></param>
/// <returns>If you should start a pattern calc now</returns>
public bool PredictNext(OfcNumber current)
{
Index++;
if (Index >= _limit)
{
return(false);
}
var next = _values[Index];
if (next.Number == current.Number && next.Exponent == current.Exponent) //same pattern
{
return(true);
}
var ahead = _values[Index + 1];
if (next.Number + (next.Number - current.Number) == ahead.Number && next.Exponent + (next.Exponent - current.Exponent) == ahead.Exponent) // offset pattern
{
return(true);
}
return(false);
}
/// <summary>
/// Reads a Single Value from its binary representation without the control bit (isBlock)
/// </summary>
/// <param name="reader"></param>
/// <param name="metadata"></param>
/// <returns></returns>
public static OfcNumber ReadSingleValueWithoutControlBit(StreamBitReader reader, BlockyMetadata metadata)
{
var value = new OfcNumber();
if (!metadata.IsAbsolute)
{
value.IsNegative = reader.ReadByte(1) > 0;
}
else
{
value.IsNegative = metadata.IsNegative;
}
value.Number = (long)reader.Read(metadata.MaxNeededBitsNumber);
var isExpNegative = reader.ReadByte(1) > 0;
value.Exponent = (short)reader.Read(metadata.MaxNeededBitsExponent); // Bug: Potentually reading exp even though NoExp is set. Same in writing method! (Ineffizient)
if (isExpNegative)
{
value.Exponent *= -1;
}
return(value);
}
public override bool ProcessValue(ref Block block, OfcNumber value, int index, ref int bitDiff)
{
if (value.Exponent != 0)
{
return(false); //Todo: maybe not impossible?
}
if (block.AbsoluteSign)
{
if (block.IsSignNegative != value.IsNegative) // Check if the new value works with the current "absolute sign" block header
{
block.AbsoluteSign = false;
bitDiff -= block.Length - 1; // We loose 1 bit per value, because we need to write down the sign now ... but we save 1 because less block header stuffs
}
else
{
bitDiff++;
}
}
if (value.Number > block.BiggestNumber) // If the biggest value in the block is smaller than the new one, we need to set it for future calculations to be correct
{
block.BiggestNumber = value.Number;
if (block.NeededBits < value.NeededBitsNumber) // If the new number needs more bits than specified in the block header, we need to adjust that
{
var nbNewBiggest = value.NeededBitsNumber;
bitDiff += block.DifferenceWithNb(Context.Metadata, ref nbNewBiggest); // Adds the difference in bits that comes with changing the block header. May change bigNumNb to the global-header-max! (can be worth, because no header then ...)
block.NeededBits = nbNewBiggest; // Need to set this after the call for future calculations to be exact
}
}
block.Length++;
bitDiff += Context.Metadata.MaxNeededBitsExponent + 1;
if (block.OverrideGlobalNb)
{
bitDiff += Context.Metadata.MaxNeededBitsNumber - block.NeededBits;
}
return(true);
}
public override bool ProcessValue(ref Block block, OfcNumber value, int index, ref int bitDiff)
{
throw new InvalidOperationException("The PingPong pattern should not be processing values, as it is created in post-compression optimisation. This CompressionMethod is just for writing!");
//todo: Dirty!
//if (value.Exponent != 0)
// return false;
////if (block.Length == 0) return true;
//var patternDiff = value.Number - Values[index - 1].Number;
//if (patternDiff != 0 && value.Number == block.PatternProperties.PatternNum1)
//{
// if (block.PatternProperties.RepeatCount == byte.MaxValue)
// return false;
// block.PatternProperties.RepeatCount++;
//}
//var patternTurnDiff = block.Length % block.PatternProperties.Length; // 0 = at end of pattern, will repeat now. else = in pattern
//if ((patternDiff != 0 && (patternTurnDiff != 0 || value.Number != block.PatternProperties.PatternNum1 && value.Number != block.PatternProperties.PatternNum2))
// || patternDiff == 0 && patternTurnDiff == 0)
////|| (block.Pattern == Block.PatternType.Increasing && patternDiff != 1))
//{
// if (index - block.Index >= byte.MaxValue) return false;
// var nb = block.NeededBits;
// block.NeededBits = 0; // the nb per value if we still had the pattern
// bitDiff += block.DifferenceWithNb(Compression, ref nb) - 2; // -2 because now we have no "Pattern" option
// block.HasPattern = false;
//}
//else
//{
// bitDiff += patternTurnDiff == 0 ? block.PatternProperties.Length * 1000 : -1000; //This is a cheap fix to make blocks with a not finished pingpongpattern "invalid". Everytime it finishes, the cost gets refunded
//}
//bitDiff += _singleValueBits;
//return true;
}
/// <summary>
/// Adds the given value to the potentual block, or sets IsValid to false if it can't be added
/// </summary>
/// <param name="value"></param>
/// <param name="index"></param>
/// <returns></returns>
public bool ProcessValue(OfcNumber value, int index)
{
return(VirtualBlock.Method.ProcessValue(ref VirtualBlock, value, index, ref SavedBits));
}
//public bool CheckIntegrity()
//{
//}
public void Write(OfcNumber value)
{
_writer.Report(value);
}
public abstract bool ProcessValue(ref Block block, OfcNumber value, int index, ref int bitDiff);
/// <summary>
/// Readjusts the numbers so that they are easier to compress. Loses some precision (of course)
/// </summary>
/// <param name="numbers"></param>
/// <param name="config"></param>
public static void Round([NotNull] List <OfcNumber> numbers, [NotNull] IConfiguaration config)
{
if (numbers.Count < 2)
{
return;
}
if (!config.Has("RoundingMin") || !config.Has("RoundingMax") || !config.Has("RoundingEpsilon"))
{
throw new ArgumentException("Invalid config numbers (Rounding)");
}
var min = config.Get <double>("RoundingMin");
var max = config.Get <double>("RoundingMax");
if (min >= max)
{
throw new ArgumentException("Invalid config numbers");
}
var minOfc = OfcNumber.Parse(min.ToString(CultureInfo.InvariantCulture));
var maxOfc = OfcNumber.Parse(max.ToString(CultureInfo.InvariantCulture));
var epsilon = config.Get <double>("RoundingEpsilon");
var epsilonTwice = epsilon * 2d;
var epsilonOfc = OfcNumber.Parse(epsilon.ToString(CultureInfo.InvariantCulture));
var currentStart = 0;
var currentMin = numbers[0].Reconstructed;
var currentMax = currentMin;
var currentMaxOfc = numbers[0];
for (var i = 1; i < numbers.Count; i++)
{
var number = numbers[i];
var numberRec = number.Reconstructed;
if (numberRec < min)
{
numbers[i] = minOfc;
}
else if (numberRec > max)
{
numbers[i] = maxOfc;
}
if (numberRec > currentMax)
{
currentMax = numberRec;
if (numberRec - currentMin > epsilonTwice)
{
if (currentStart != i - 1)
{
AdjustNumbers(numbers, currentStart, i, currentMaxOfc, epsilonOfc);
}
currentStart = i;
currentMin = numberRec;
currentMaxOfc = number;
}
}
if (numberRec < currentMin)
{
currentMin = numberRec;
if (currentMax - numberRec > epsilonTwice)
{
if (currentStart != i - 1)
{
AdjustNumbers(numbers, currentStart, i, currentMaxOfc, epsilonOfc);
}
currentStart = i;
currentMax = numberRec;
currentMaxOfc = number;
}
}
}
//using (var writer = new StreamWriter(new FileStream(new Random().Next(0, 100) + "raw123.txt", FileMode.OpenOrCreate)))
//{
// for (var i = 0; i < numbers.Count; i++)
// {
// writer.WriteLine(numbers[i].Reconstructed);
// }
//}
}
/// <summary>
/// Sets all values in the given List within the range from index to index2 to currentMax - epsilonOfc
/// </summary>
/// <param name="numbers"></param>
/// <param name="index"></param>
/// <param name="index2"></param>
/// <param name="currentMax"></param>
/// <param name="epsilonOfc"></param>
private static void AdjustNumbers(List <OfcNumber> numbers, int index, int index2, OfcNumber currentMax, OfcNumber epsilonOfc)
{
var allNum = currentMax - epsilonOfc;
for (var j = index; j < index2; j++)
{
numbers[j] = allNum;
}
}
public void HandleListEntry(string value)
{
_compress.Report(OfcNumber.Parse(value));
}
public override bool ProcessValue(ref Block block, OfcNumber value, int index, ref int bitDiff)
{
if (block.Length == byte.MaxValue)
{
return(false);
}
if (block.Length == 0)
{
block.Length++;
bitDiff += Context.Headers.StandardSingleValue;
return(true);
}
var patternDiff = value.Number - Values[index - 1].Number != 0 || value.Exponent - Values[index - 1].Exponent != 0;
if (patternDiff)
{
if (block.Length == 1)
{
block.Length++;
block.Pattern = Block.PatternType.Offset;
bitDiff += Context.Headers.StandardBlockPatternSame - Context.Headers.StandardBlockPatternOffset;
block.Method = Context.GetInitializedMethod(Blockfinding.Blockfinding.Methods.PatternOffset);
return(true);
}
var firstValue = Values[block.Index];
//bitDiff -= _singleValueBits * block.Length - block.Length * MaxNeededBitsExponent * 2;
block.HasPattern = false;
block.BiggestNumber = Math.Max(firstValue.Number, value.Number);
if (firstValue.IsNegative == value.IsNegative)
{
block.AbsoluteSign = true;
block.IsSignNegative = value.IsNegative;
//bitDiff--; commented this out, as the isNegative is now in the default header!
}
else
{
bitDiff -= block.Length;
}
var nb = block.BiggestNumber.GetNeededBits();
block.NeededBits = 0; // the nb per value if we still had the pattern
bitDiff += block.DifferenceWithNb(Context.Metadata, ref nb);
block.NeededBits = nb;
if (firstValue.Exponent == 0 && value.Exponent == 0)
{
block.Length++;
bitDiff += Context.Headers.StandardBlockPatternSame - Context.Headers.StandardBlockNumbersNoExp;
block.Method = Context.GetInitializedMethod(Blockfinding.Blockfinding.Methods.NumbersNoExp);
return(true);
}
var oldMethod = block.Method;
block.Method = Context.GetInitializedMethod(Blockfinding.Blockfinding.Methods.FloatSimmilar);
block.Exponent = firstValue.Exponent;
block.HasExponent = true;
bitDiff += Context.Headers.StandardBlockPatternSame - Context.Headers.StandardBlockFloatSimmilar;
if (firstValue.Exponent == value.Exponent)
{
block.Length++;
return(true);
}
var result = block.Method.ProcessValue(ref block, value, index, ref bitDiff);
if (!result)
{
block.HasPattern = true;
block.Method = oldMethod;
}
return(result);
}
block.Length++;
bitDiff += Context.Headers.StandardSingleValue;
return(true);
}
public void Write(OfcNumber value)
{
Values[_index++] = value;
}
public override bool ProcessValue(ref Block block, OfcNumber value, int index, ref int bitDiff)
{
if (block.Length == byte.MaxValue)
{
return(false);
}
if (block.Exponent < value.Exponent) // The exp is bigger, which means we will set the exp equal and add stuff to the value to balance things
{
var expDiff = value.Exponent - block.Exponent;
if (expDiff > 18)
{
return(false); // long can't hold 10^19 or higher
}
var multiplier = (long)Math.Pow(10, expDiff);
var newNum = value.Number;
if (value.Number != 0)
{
if ((long)Context.Metadata.LargestPossibleValue / value.Number < multiplier)
{
return(false); // overflow check
}
newNum = value.Number * multiplier; // This balances the expDiff we subtraced from Exponent
}
var newNb = newNum.GetNeededBits(); // Recalculate the needed bits for the number, because that may have changed
if (value.NeededBitsNumber > Context.Metadata.MaxNeededBitsNumber)
{
return(false); //Todo: you can't write with more than max bits (even in blocks when the nb are overridden), because of "nbnbmaxnumber" ... maybe we can work our way around that? Currently we just throw the block away ...
}
// We don't actually apply the changes before checking if we can even write the new value ...
value.Exponent -= (short)expDiff;
value.NeededBitsNumber = newNb;
value.Number = newNum;
if (value.NeededBitsNumber > block.NeededBits) // Value cannot be added to block, we have to check what would change if we change the NeededBits in the block header!
{
var nbNewNumber = value.NeededBitsNumber;
bitDiff += block.DifferenceWithNb(Context.Metadata, ref nbNewNumber); // Adds the difference in bits if we would change the block header. May change nbNewNumber to the global-header-max! (can be worth, because no header then ...)
block.NeededBits = nbNewNumber; // Need to set this after the call for future calculations to be exact
}
// Values[index] = value;
}
else if (block.Exponent > value.Exponent) // The exp is smaller, which means we will recalculate the whole block to fit this value
{
var expDiff = (block.Exponent - value.Exponent);
if (expDiff > 18)
{
return(false); // long can't hold 10^19 or higher
}
var multiplier = (long)Math.Pow(10, expDiff);
if (block.BiggestNumber > 0 && (long)Context.Metadata.LargestPossibleValue / block.BiggestNumber < multiplier)
{
return(false); // overflow check
}
var newNum = block.BiggestNumber * multiplier; // Recalculate the biggest number of the blog
block.BiggestNumber = Math.Max(value.Number, newNum); // value.Number > newNum ? value.Number : newNum; // Recalculate the biggest number of the blog
block.Exponent = value.Exponent; // Setting the new exponent for the whole block
var bigNumNb = Math.Max(block.BiggestNumber.GetNeededBits(), value.NeededBitsNumber); // Math.Max because the new value could be bigger than the oldest biggest value, which would give a wrong result
if (bigNumNb > Context.Metadata.MaxNeededBitsNumber)
{
return(false); //Todo: you can't write with more than max bits (even in blocks when the nb are overridden), because of "nbnbmaxnumber" ... maybe we can work our way around that? Currently we just throw the block away ...
}
if (bigNumNb > block.NeededBits) // The change in exp made the numbers bigger -> we need more Nb to store the numbers
{
var diff = block.DifferenceWithNb(Context.Metadata, ref bigNumNb); // Adds the difference in bits that comes with changing the block header. May change bigNumNb to the global-header-max! (can be worth, because no header then ...)
bitDiff += diff;
block.NeededBits = bigNumNb; // Need to set this after the call for future calculations to be exact
}
}
if (block.AbsoluteSign) // Check if the new value works with the current "absolute sign" block header
{
if (block.IsSignNegative != value.IsNegative)
{
block.AbsoluteSign = false;
bitDiff -= block.Length - 1; // We loose 1 bit per value, because we need to write down the sign now ... but we save 1 because less block header stuffs
}
else
{
bitDiff++;
}
}
if (value.Number > block.BiggestNumber) // If the biggest value in the block is smaller than the new one, we need to set it for future calculations to be correct
{
block.BiggestNumber = value.Number;
if (block.NeededBits < value.NeededBitsNumber) // If the new number needs more bits than specified in the block header, we need to adjust that
{
var nbNewBiggest = value.NeededBitsNumber;
bitDiff += block.DifferenceWithNb(Context.Metadata, ref nbNewBiggest); // Adds the difference in bits that comes with changing the block header. May change bigNumNb to the global-header-max! (can be worth, because no header then ...)
block.NeededBits = nbNewBiggest; // Need to set this after the call for future calculations to be exact
}
}
block.Length++;
bitDiff += Context.Metadata.MaxNeededBitsExponent + 2;
if (block.OverrideGlobalNb)
{
bitDiff += Context.Metadata.MaxNeededBitsNumber - block.NeededBits;
}
return(true);
}
