private static void TestByteRead(uint delta)
{
int loop = 10000;
byte[] buffer = new byte[65535];
int position = 0;
for (int x = 0; x < 200; x++)
{
Encoding7Bit.Write(buffer, ref position, delta);
Encoding7Bit.Write(buffer, ref position, delta);
Encoding7Bit.Write(buffer, ref position, delta);
Encoding7Bit.Write(buffer, ref position, delta);
Encoding7Bit.Write(buffer, ref position, delta);
}
Stopwatch sw = new Stopwatch();
sw.Start();
for (int repeat = 0; repeat < loop; repeat++)
{
position = 0;
for (int x = 0; x < 200; x++)
{
delta = Encoding7Bit.ReadUInt32(buffer, ref position);
delta = Encoding7Bit.ReadUInt32(buffer, ref position);
delta = Encoding7Bit.ReadUInt32(buffer, ref position);
delta = Encoding7Bit.ReadUInt32(buffer, ref position);
delta = Encoding7Bit.ReadUInt32(buffer, ref position);
}
}
sw.Stop();
//MessageBox.Show(position.ToString() + "b " + (loop * 1000 / sw.Elapsed.TotalSeconds / 1000000).ToString());
}
/// <summary>
/// Writes the supplied <paramref name="value"/> to
/// <paramref name="stream"/> along with prefixing the length
/// so it can be properly read as a unit.
/// </summary>
/// <param name="stream">the stream to write to</param>
/// <param name="value">the value to write</param>
public static void WriteWithLength(this Stream stream, byte[] value)
{
Encoding7Bit.Write(stream.WriteByte, (uint)value.Length);
if (value.Length > 0)
{
stream.Write(value, 0, value.Length);
}
}
private uint DecodeQuality(int code, TsscPointMetadata nextPoint)
{
uint quality = code == TsscCodeWords.Quality2 ?
nextPoint.PrevQuality2 :
Encoding7Bit.ReadUInt32(m_data, ref m_position);
nextPoint.PrevQuality2 = nextPoint.PrevQuality1;
nextPoint.PrevQuality1 = quality;
return(quality);
}
public override void Write7Bit(uint value)
{
const int size = 5;
if (Current + size <= LastWrite)
{
Current += Encoding7Bit.Write(Current, value);
return;
}
base.Write7Bit(value);
}
private long DecodeTimestamp(int code)
{
long timestamp = code switch
{
TsscCodeWords.TimeDelta1Forward => m_prevTimestamp1 + m_prevTimeDelta1,
TsscCodeWords.TimeDelta2Forward => m_prevTimestamp1 + m_prevTimeDelta2,
TsscCodeWords.TimeDelta3Forward => m_prevTimestamp1 + m_prevTimeDelta3,
TsscCodeWords.TimeDelta4Forward => m_prevTimestamp1 + m_prevTimeDelta4,
TsscCodeWords.TimeDelta1Reverse => m_prevTimestamp1 - m_prevTimeDelta1,
TsscCodeWords.TimeDelta2Reverse => m_prevTimestamp1 - m_prevTimeDelta2,
TsscCodeWords.TimeDelta3Reverse => m_prevTimestamp1 - m_prevTimeDelta3,
TsscCodeWords.TimeDelta4Reverse => m_prevTimestamp1 - m_prevTimeDelta4,
TsscCodeWords.Timestamp2 => m_prevTimestamp2,
_ => m_prevTimestamp1 ^ (long)Encoding7Bit.ReadUInt64(m_data, ref m_position)
};
//Save the smallest delta time
long minDelta = Math.Abs(m_prevTimestamp1 - timestamp);
if (minDelta < m_prevTimeDelta4 && minDelta != m_prevTimeDelta1 && minDelta != m_prevTimeDelta2 && minDelta != m_prevTimeDelta3)
{
if (minDelta < m_prevTimeDelta1)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = m_prevTimeDelta2;
m_prevTimeDelta2 = m_prevTimeDelta1;
m_prevTimeDelta1 = minDelta;
}
else if (minDelta < m_prevTimeDelta2)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = m_prevTimeDelta2;
m_prevTimeDelta2 = minDelta;
}
else if (minDelta < m_prevTimeDelta3)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = minDelta;
}
else
{
m_prevTimeDelta4 = minDelta;
}
}
m_prevTimestamp2 = m_prevTimestamp1;
m_prevTimestamp1 = timestamp;
return(timestamp);
}
private void WriteQualityChange(uint quality, TsscPointMetadata point)
{
if (point.PrevQuality2 == quality)
{
m_lastPoint.WriteCode(TsscCodeWords.Quality2);
}
else
{
m_lastPoint.WriteCode(TsscCodeWords.Quality7Bit32);
Encoding7Bit.Write(m_data, ref m_position, quality);
}
point.PrevQuality2 = point.PrevQuality1;
point.PrevQuality1 = quality;
}
private static void TestMethod15()
{
byte[] buffer = new byte[2];
buffer[1] = 23;
for (short x = 0; x > 0; x++)
{
int pos = 0;
Encoding7Bit.WriteInt15(buffer, ref pos, x);
if (x < 128)
{
if (pos != 1)
{
throw new Exception();
}
}
else
{
if (pos != 2)
{
throw new Exception();
}
}
pos = 0;
short v = Encoding7Bit.ReadInt15(buffer, ref pos);
if (x < 128)
{
if (pos != 1)
{
throw new Exception();
}
}
else
{
if (pos != 2)
{
throw new Exception();
}
}
if (x != v)
{
throw new Exception();
}
}
}
private static unsafe void TestRandomGenerated()
{
Random rand = new Random();
int seed = rand.Next();
rand = new Random(seed);
byte[] buffer = new byte[10];
byte[] data = new byte[4];
fixed(byte *lp = data)
{
for (int x = 0; x < 100000; x++)
{
uint value, result;
int position = 1;
rand.NextBytes(data);
data[1] = 0;
data[2] = 0;
data[3] = 0;
value = *(uint *)lp;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt32(buffer, ref position);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
}
for (int x = 0; x < 100000; x++)
{
uint value, result;
int position = 1;
rand.NextBytes(data);
data[2] = 0;
data[3] = 0;
value = *(uint *)lp;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt32(buffer, ref position);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
}
for (int x = 0; x < 100000; x++)
{
uint value, result;
int position = 1;
rand.NextBytes(data);
data[3] = 0;
value = *(uint *)lp;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt32(buffer, ref position);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
}
for (int x = 0; x < 100000; x++)
{
uint value, result;
int position = 1;
rand.NextBytes(data);
value = *(uint *)lp;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt32(buffer, ref position);
if (position != 1 + Encoding7Bit.GetSize(value))
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
}
}
}
public override unsafe int Decode(byte *stream, HistorianKey prevKey, HistorianValue prevValue, HistorianKey key, HistorianValue value, out bool isEndOfStream)
{
isEndOfStream = false;
int size = 0;
uint code = stream[0];
//Compression Stages:
// Stage 1: Big Positive Float.
// Stage 2: Big Negative Float.
// Stage 3: Zero
// Stage 4: 32 bit
// Stage 5: Catch all
if (code < 0x80)
{
//If stage 1 (50% success)
key.Timestamp = prevKey.Timestamp;
key.PointID = prevKey.PointID + 1 + ((code >> 4) & 0x7);
key.EntryNumber = 0;
value.Value1 = (4u << 28) | (code & 0xF) << 24 | (uint)stream[1] << 16 | (uint)stream[2] << 8 | (uint)stream[3] << 0;
value.Value2 = 0;
value.Value3 = 0;
return(4);
}
if (code < 0xC0)
{
//If stage 2 (16% success)
key.Timestamp = prevKey.Timestamp;
key.PointID = prevKey.PointID + 1 + ((code >> 4) & 0x3);
key.EntryNumber = 0;
value.Value1 = (12u << 28) | (code & 0xF) << 24 | (uint)stream[1] << 16 | (uint)stream[2] << 8 | (uint)stream[3] << 0;
value.Value2 = 0;
value.Value3 = 0;
return(4);
}
if (code < 0xD0)
{
//If stage 3 (28% success)
key.Timestamp = prevKey.Timestamp;
key.PointID = prevKey.PointID + 1 + (code & 0xF);
key.EntryNumber = 0;
value.Value1 = 0;
value.Value2 = 0;
value.Value3 = 0;
return(1);
}
if (code < 0xE0)
{
//If stage 4 (3% success)
key.Timestamp = prevKey.Timestamp;
key.PointID = prevKey.PointID + 1 + (code & 0xF);
key.EntryNumber = 0;
value.Value1 = *(uint *)(stream + 1);
value.Value2 = 0;
value.Value3 = 0;
return(5);
}
//Stage 5: 2%
//Stage 5: Catch All
size = 1;
if ((code & 16) != 0) //T is set
{
key.Timestamp = prevKey.Timestamp + Encoding7Bit.ReadUInt64(stream, ref size);
key.PointID = Encoding7Bit.ReadUInt64(stream, ref size);
}
else
{
key.Timestamp = prevKey.Timestamp;
key.PointID = prevKey.PointID + Encoding7Bit.ReadUInt64(stream, ref size);
}
if ((code & 8) != 0) //E is set)
{
key.EntryNumber = Encoding7Bit.ReadUInt64(stream, ref size);
}
else
{
key.EntryNumber = 0;
}
if ((code & 4) != 0) //V1 is set)
{
value.Value1 = *(ulong *)(stream + size);
size += 8;
}
else
{
value.Value1 = *(uint *)(stream + size);
size += 4;
}
if ((code & 2) != 0) //V2 is set)
{
value.Value2 = Encoding7Bit.ReadUInt64(stream, ref size);
}
else
{
value.Value2 = 0;
}
if ((code & 1) != 0) //V3 is set)
{
value.Value3 = Encoding7Bit.ReadUInt64(stream, ref size);
}
else
{
value.Value3 = 0;
}
return(size);
}
public override unsafe int Encode(byte *stream, HistorianKey prevKey, HistorianValue prevValue, HistorianKey key, HistorianValue value)
{
//ToDo: Make stage 1 still work on big endian processors.
int size = 0;
//Compression Stages:
// Stage 1: Big Positive Float.
// Stage 2: Big Negative Float.
// Stage 3: Zero
// Stage 4: 32 bit
// Stage 5: Catch all
if (key.Timestamp == prevKey.Timestamp &&
key.PointID > prevKey.PointID && (key.PointID - prevKey.PointID <= 16) &&
key.EntryNumber == 0 &&
value.Value1 <= uint.MaxValue && //must be a 32-bit value
value.Value2 == 0 &&
value.Value3 == 0)
{
uint deltaPointId = (uint)(key.PointID - prevKey.PointID);
//Could match Stage 1, 2, 3, or 4
//Check for Stage 3
if (value.Value1 == 0 && deltaPointId <= 16)
{
//Stage 3: 28% of the time.
stream[0] = (byte)(0xC0 | (deltaPointId - 1));
return(1);
}
//Check for Stage 1
if ((value.Value1 >> 28) == 4 && deltaPointId <= 8)
{
//Stage 1: 46% of the time
//Big Positive Float
//Must be stored big endian
//ByteCode is 0DDDVVVV
stream[0] = (byte)(((value.Value1 >> 24) & 0xF) | (deltaPointId - 1) << 4);
stream[1] = (byte)(value.Value1 >> 16);
stream[2] = (byte)(value.Value1 >> 8);
stream[3] = (byte)value.Value1;
return(4);
}
//Check for stage 2
if ((value.Value1 >> 28) == 12 && deltaPointId <= 4)
{
//Must be stored big endian
//ByteCode is 10DDVVVV
stream[0] = (byte)(0x80 | ((value.Value1 >> 24) & 0xF) | (deltaPointId - 1) << 4);
stream[1] = (byte)(value.Value1 >> 16);
stream[2] = (byte)(value.Value1 >> 8);
stream[3] = (byte)value.Value1;
return(4);
}
//Check for stage 4
//All conditions are in the logic statement that enters this block.
// deltaPointID <= 16
stream[0] = (byte)(0xD0 | (deltaPointId - 1));
*(uint *)(stream + 1) = (uint)value.Value1;
return(5);
}
//Stage 5: Catch All
stream[0] = 0xE0;
size = 1;
if (key.Timestamp != prevKey.Timestamp)
{
stream[0] |= 0x10; //Set bit T
Encoding7Bit.Write(stream, ref size, key.Timestamp - prevKey.Timestamp);
Encoding7Bit.Write(stream, ref size, key.PointID);
}
else
{
Encoding7Bit.Write(stream, ref size, key.PointID - prevKey.PointID);
}
if (key.EntryNumber != 0)
{
stream[0] |= 0x08; //Set bit E
Encoding7Bit.Write(stream, ref size, key.EntryNumber);
}
if (value.Value1 > uint.MaxValue)
{
stream[0] |= 0x04; //Set Bit V1
*(ulong *)(stream + size) = value.Value1;
size += 8;
}
else
{
*(uint *)(stream + size) = (uint)value.Value1;
size += 4;
}
if (value.Value2 != 0)
{
stream[0] |= 0x02; //Set Bit V2
Encoding7Bit.Write(stream, ref size, value.Value2);
}
if (value.Value3 != 0)
{
//ToDo: Special encoding of flag fields
stream[0] |= 0x01; //Set Bit V3
Encoding7Bit.Write(stream, ref size, value.Value3);
}
return(size);
}
private void WriteTimestampChange(long timestamp)
{
if (m_prevTimestamp2 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.Timestamp2);
}
else if (m_prevTimestamp1 < timestamp)
{
if (m_prevTimestamp1 + m_prevTimeDelta1 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta1Forward);
}
else if (m_prevTimestamp1 + m_prevTimeDelta2 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta2Forward);
}
else if (m_prevTimestamp1 + m_prevTimeDelta3 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta3Forward);
}
else if (m_prevTimestamp1 + m_prevTimeDelta4 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta4Forward);
}
else
{
m_lastPoint.WriteCode(TsscCodeWords.TimeXOR7Bit);
Encoding7Bit.Write(m_data, ref m_position, (ulong)(timestamp ^ m_prevTimestamp1));
}
}
else
{
if (m_prevTimestamp1 - m_prevTimeDelta1 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta1Reverse);
}
else if (m_prevTimestamp1 - m_prevTimeDelta2 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta2Reverse);
}
else if (m_prevTimestamp1 - m_prevTimeDelta3 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta3Reverse);
}
else if (m_prevTimestamp1 - m_prevTimeDelta4 == timestamp)
{
m_lastPoint.WriteCode(TsscCodeWords.TimeDelta4Reverse);
}
else
{
m_lastPoint.WriteCode(TsscCodeWords.TimeXOR7Bit);
Encoding7Bit.Write(m_data, ref m_position, (ulong)(timestamp ^ m_prevTimestamp1));
}
}
//Save the smallest delta time
long minDelta = Math.Abs(m_prevTimestamp1 - timestamp);
if (minDelta < m_prevTimeDelta4 &&
minDelta != m_prevTimeDelta1 &&
minDelta != m_prevTimeDelta2 &&
minDelta != m_prevTimeDelta3)
{
if (minDelta < m_prevTimeDelta1)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = m_prevTimeDelta2;
m_prevTimeDelta2 = m_prevTimeDelta1;
m_prevTimeDelta1 = minDelta;
}
else if (minDelta < m_prevTimeDelta2)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = m_prevTimeDelta2;
m_prevTimeDelta2 = minDelta;
}
else if (minDelta < m_prevTimeDelta3)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = minDelta;
}
else
{
m_prevTimeDelta4 = minDelta;
}
}
m_prevTimestamp2 = m_prevTimestamp1;
m_prevTimestamp1 = timestamp;
}
public unsafe void AddMeasurement(ushort id, long timestamp, uint quality, float value)
{
if (!CanAddMeasurements)
{
throw new Exception("Not enough buffer space to add a new measurement.");
}
byte[] buffer = m_buffer;
int index = m_index;
PointMetaData point;
if (!m_points.TryGetValue(id, out point))
{
m_lastMeasurementHeaderIndex = -1;
point = new PointMetaData();
point.PointID = m_points.Count;
m_lastPoint.ExpectedNextPointID = point.PointID;
m_points.Add(id, point);
buffer[index++] = 5;
Buffer.BlockCopy(BitConverter.GetBytes(id), 0, buffer, index, 2);
index += 2;
}
byte code = 0;
if (m_lastPoint.ExpectedNextPointID != point.PointID)
{
code |= 32;
m_lastPoint.ExpectedNextPointID = point.PointID;
}
if (quality != point.LastQuality)
{
code |= 16;
point.LastQuality = quality;
}
if (timestamp != m_previousTimestamp)
{
code |= 8;
}
uint bitsChanged = (*(uint *)&value) ^ point.LastValue;
point.LastValue ^= bitsChanged;
if (bitsChanged > 0xFFFFFFu)
{
code |= 4;
}
else if (bitsChanged > 0xFFFFu)
{
code |= 3;
}
else if (bitsChanged > 0xFFu)
{
code |= 2;
}
else if (bitsChanged > 0u)
{
code |= 1;
}
else
{
code |= 0;
}
// If the computed code is the same as what is expected based on the last measurement,
// and there are enough bits available in the last encoded header to skip writing this code then do so.
if (m_lastPoint.ExpectedNextCode == code && m_lastMeasurementHeaderIndex >= 0)
{
// Increment the previous code to increase the run length of headers that don't need to be changed.
buffer[m_lastMeasurementHeaderIndex] += 64;
if (buffer[m_lastMeasurementHeaderIndex] >= 192)
{
m_lastMeasurementHeaderIndex = -1;
}
}
else
{
// Write the code to the stream and updated expected values.
m_lastMeasurementHeaderIndex = index;
m_lastPoint.ExpectedNextCode = code;
buffer[index++] = code;
}
if ((code & 32) != 0)
{
Encoding7Bit.Write(buffer, ref index, (uint)point.PointID);
}
if ((code & 16) != 0)
{
buffer[index++] = (byte)quality;
buffer[index++] = (byte)(quality >> 8);
buffer[index++] = (byte)(quality >> 16);
buffer[index++] = (byte)(quality >> 24);
}
if (bitsChanged > 0xFFFFFFu)
{
buffer[index++] = (byte)bitsChanged;
buffer[index++] = (byte)(bitsChanged >> 8);
buffer[index++] = (byte)(bitsChanged >> 16);
buffer[index++] = (byte)(bitsChanged >> 24);
}
else if (bitsChanged > 0xFFFFu)
{
buffer[index++] = (byte)bitsChanged;
buffer[index++] = (byte)(bitsChanged >> 8);
buffer[index++] = (byte)(bitsChanged >> 16);
}
else if (bitsChanged > 0xFFu)
{
buffer[index++] = (byte)bitsChanged;
buffer[index++] = (byte)(bitsChanged >> 8);
}
else if (bitsChanged > 0u)
{
buffer[index++] = (byte)bitsChanged;
}
if (timestamp != m_previousTimestamp)
{
Encoding7Bit.Write(buffer, ref index, (ulong)(timestamp ^ m_previousTimestamp));
m_previousTimestamp = timestamp;
}
m_lastPoint = point;
m_index = index;
}
/// <summary>
/// Reads from the underlying stream in little endian format. Advancing the position.
/// </summary>
/// <returns>The data read</returns>
public virtual uint Read7BitUInt32()
{
return(Encoding7Bit.ReadUInt32(ReadUInt8));
}
/// <summary>
/// Reads from the underlying stream in little endian format. Advancing the position.
/// </summary>
/// <returns>The data read</returns>
public virtual ulong Read7BitUInt64()
{
return(Encoding7Bit.ReadUInt64(ReadUInt8));
}
public DecompressionExitCode GetMeasurement()
{
TryAgain:
PointMetaData nextPoint = null;
if (m_buffer.Position == m_length && m_bitStream.IsEmpty)
{
return(DecompressionExitCode.EndOfStreamOccured);
}
int code = m_lastPoint.ReadCode(m_bitStream);
if (code >= MeasurementStreamCodes.NewPointId && code <= MeasurementStreamCodes.FlushBits)
{
if (code == MeasurementStreamCodes.NewPointId)
{
MeasurementTypeCode type = (MeasurementTypeCode)m_buffer.Data[m_buffer.Position++];
PointMetaData point;
switch (type)
{
case MeasurementTypeCode.Int32:
case MeasurementTypeCode.UInt32:
case MeasurementTypeCode.Single:
point = new PointMetaDataInt32(m_buffer, m_bitStream, type, m_points.Count);
break;
case MeasurementTypeCode.UInt64:
case MeasurementTypeCode.Int64:
case MeasurementTypeCode.Double:
point = new PointMetaDataInt64(m_buffer, m_bitStream, type, m_points.Count);
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
NewMeasurementRegisteredTypeCode = type;
NewMeasurementRegisteredId = point.ReferenceId;
m_lastPoint.PrevNextPointId1 = point.ReferenceId;
m_points.Add(point);
NewMeasurementRegisteredMetadataLength = (int)Encoding7Bit.ReadUInt32(m_buffer.Data, ref m_buffer.Position);
while (NewMeasurementRegisteredMetadata.Length < NewMeasurementRegisteredMetadataLength)
{
NewMeasurementRegisteredMetadata = new byte[NewMeasurementRegisteredMetadata.Length * 2];
}
Array.Copy(m_buffer.Data, m_buffer.Position, NewMeasurementRegisteredMetadata, 0, NewMeasurementRegisteredMetadataLength);
m_buffer.Position += NewMeasurementRegisteredMetadataLength;
return(DecompressionExitCode.NewMeasurementRegistered);
}
else if (code == MeasurementStreamCodes.UserCommand)
{
UserCommand = (long)Encoding7Bit.ReadUInt64(m_buffer.Data, ref m_buffer.Position);
return(DecompressionExitCode.UserData);
}
else if (code == MeasurementStreamCodes.UserCommandWithData)
{
UserCommand = (long)Encoding7Bit.ReadUInt64(m_buffer.Data, ref m_buffer.Position);
UserCommandDataLength = (int)Encoding7Bit.ReadUInt32(m_buffer.Data, ref m_buffer.Position);
while (UserCommandData.Length < UserCommandDataLength)
{
UserCommandData = new byte[UserCommandData.Length * 2];
}
Array.Copy(m_buffer.Data, m_buffer.Position, UserCommandData, 0, UserCommandDataLength);
m_buffer.Position += UserCommandDataLength;
return(DecompressionExitCode.UserDataWithValue);
}
else if (code == MeasurementStreamCodes.FlushBits)
{
m_bitStream.Clear();
goto TryAgain;
}
else
{
throw new Exception("Programming Error.");
}
}
if (code < MeasurementStreamCodes.PointIDXOR4)
{
throw new Exception("Expecting higher code");
}
if (code <= MeasurementStreamCodes.PointIDXOR32)
{
if (code == MeasurementStreamCodes.PointIDXOR4)
{
m_lastPoint.PrevNextPointId1 ^= m_bitStream.ReadBits4();
}
else if (code == MeasurementStreamCodes.PointIDXOR8)
{
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++];
}
else if (code == MeasurementStreamCodes.PointIDXOR12)
{
m_lastPoint.PrevNextPointId1 ^= m_bitStream.ReadBits4();
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 4;
}
else if (code == MeasurementStreamCodes.PointIDXOR16)
{
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++];
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 8;
}
else if (code == MeasurementStreamCodes.PointIDXOR20)
{
m_lastPoint.PrevNextPointId1 ^= m_bitStream.ReadBits4();
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 4;
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 12;
}
else if (code == MeasurementStreamCodes.PointIDXOR24)
{
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++];
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 8;
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 16;
}
else if (code == MeasurementStreamCodes.PointIDXOR32)
{
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++];
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 8;
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 16;
m_lastPoint.PrevNextPointId1 ^= m_buffer.Data[m_buffer.Position++] << 24;
}
else
{
throw new Exception("Programming Error.");
}
code = m_lastPoint.ReadCode(m_bitStream);
}
if (code < MeasurementStreamCodes.TimeDelta1Forward)
{
throw new Exception("Expecting higher code");
}
ID = m_lastPoint.PrevNextPointId1;
nextPoint = m_points[m_lastPoint.PrevNextPointId1];
Quality = nextPoint.PrevQuality1;
if (code <= MeasurementStreamCodes.TimeXOR7Bit)
{
if (code == MeasurementStreamCodes.TimeDelta1Forward)
{
Timestamp = m_prevTimestamp1 + m_prevTimeDelta1;
}
else if (code == MeasurementStreamCodes.TimeDelta2Forward)
{
Timestamp = m_prevTimestamp1 + m_prevTimeDelta2;
}
else if (code == MeasurementStreamCodes.TimeDelta3Forward)
{
Timestamp = m_prevTimestamp1 + m_prevTimeDelta3;
}
else if (code == MeasurementStreamCodes.TimeDelta4Forward)
{
Timestamp = m_prevTimestamp1 + m_prevTimeDelta4;
}
else if (code == MeasurementStreamCodes.TimeDelta1Reverse)
{
Timestamp = m_prevTimestamp1 - m_prevTimeDelta1;
}
else if (code == MeasurementStreamCodes.TimeDelta2Reverse)
{
Timestamp = m_prevTimestamp1 - m_prevTimeDelta2;
}
else if (code == MeasurementStreamCodes.TimeDelta3Reverse)
{
Timestamp = m_prevTimestamp1 - m_prevTimeDelta3;
}
else if (code == MeasurementStreamCodes.TimeDelta4Reverse)
{
Timestamp = m_prevTimestamp1 - m_prevTimeDelta4;
}
else if (code == MeasurementStreamCodes.Timestamp2)
{
Timestamp = m_prevTimestamp2;
}
else if (code == MeasurementStreamCodes.TimeXOR7Bit)
{
Timestamp = m_prevTimestamp1 ^ (long)Encoding7Bit.ReadUInt64(m_buffer.Data, ref m_buffer.Position);
}
else
{
throw new Exception("Programming Error.");
}
//Save the smallest delta time
long minDelta = Math.Abs(m_prevTimestamp1 - Timestamp);
if (minDelta < m_prevTimeDelta4 && minDelta != m_prevTimeDelta1 && minDelta != m_prevTimeDelta2 && minDelta != m_prevTimeDelta3)
{
if (minDelta < m_prevTimeDelta1)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = m_prevTimeDelta2;
m_prevTimeDelta2 = m_prevTimeDelta1;
m_prevTimeDelta1 = minDelta;
}
else if (minDelta < m_prevTimeDelta2)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = m_prevTimeDelta2;
m_prevTimeDelta2 = minDelta;
}
else if (minDelta < m_prevTimeDelta3)
{
m_prevTimeDelta4 = m_prevTimeDelta3;
m_prevTimeDelta3 = minDelta;
}
else
{
m_prevTimeDelta4 = minDelta;
}
}
m_prevTimestamp2 = m_prevTimestamp1;
m_prevTimestamp1 = Timestamp;
code = m_lastPoint.ReadCode(m_bitStream);
}
else
{
Timestamp = m_prevTimestamp1;
}
if (code < MeasurementStreamCodes.Quality2)
{
throw new Exception("Expecting higher code");
}
if (code <= MeasurementStreamCodes.Quality7Bit32)
{
if (code == MeasurementStreamCodes.Quality2)
{
Quality = nextPoint.PrevQuality2;
}
else if (code == MeasurementStreamCodes.Quality7Bit32)
{
Quality = Encoding7Bit.ReadUInt32(m_buffer.Data, ref m_buffer.Position);
}
nextPoint.PrevQuality2 = nextPoint.PrevQuality1;
nextPoint.PrevQuality1 = Quality;
code = m_lastPoint.ReadCode(m_bitStream);
}
else
{
Quality = nextPoint.PrevQuality1;
}
if (code < 32)
{
throw new Exception("Programming Error. Expecting a value quality code.");
}
nextPoint.ReadValue(code, Value);
Value.Code = nextPoint.Code;
m_lastPoint = nextPoint;
return(DecompressionExitCode.MeasurementRead);
}
/// <summary>
/// Writes the specifed <see cref="value"/> to the underlying stream in little-endian format.
/// </summary>
/// <param name="value">the value to write</param>
public virtual void Write7Bit(ulong value)
{
Encoding7Bit.Write(Write, value);
}
/// <summary>
/// This procedure will attempt to read all of the data out of the file allocation table
/// If the file allocation table is corrupt, an error will be generated.
/// </summary>
/// <param name="buffer">the block that contains the buffer data.</param>
private void LoadFromBuffer(byte[] buffer)
{
ValidateBlock(buffer);
m_blockSize = buffer.Length;
MemoryStream stream = new MemoryStream(buffer);
BinaryReader dataReader = new BinaryReader(stream);
if (!dataReader.ReadBytes(26).SequenceEqual(FileAllocationTableHeaderBytes))
{
throw new Exception("This file is not an archive file system, or the file is corrupt, or this file system major version is not recgonized by this version of the historian");
}
char endian = (char)dataReader.ReadByte();
if (BitConverter.IsLittleEndian)
{
if (endian != 'L')
{
throw new Exception("This archive file was not writen with a little endian processor");
}
}
else
{
if (endian != 'B')
{
throw new Exception("This archive file was not writen with a big endian processor");
}
}
byte blockSizePower = dataReader.ReadByte();
if (blockSizePower > 30 || blockSizePower < 5)
{
throw new Exception("Block size of this file is not supported");
}
int blockSize = 1 << blockSizePower;
if (m_blockSize != blockSize)
{
throw new Exception("Block size is unexpected");
}
m_minimumReadVersion = dataReader.ReadInt16();
m_minimumWriteVersion = dataReader.ReadInt16();
if (!CanRead)
{
throw new Exception("The version of this file system is not recgonized");
}
m_headerVersion = m_minimumWriteVersion;
if (m_headerVersion < 0)
{
throw new Exception("Header version not supported");
}
if (m_headerVersion == 0 || m_headerVersion == 1)
{
m_isSimplifiedFileFormat = false;
m_headerBlockCount = 10;
LoadHeaderV0V1(dataReader);
return;
}
m_headerVersion = dataReader.ReadInt16();
byte fileMode = dataReader.ReadByte();
if (fileMode == 1)
{
m_isSimplifiedFileFormat = false;
}
else if (fileMode == 2)
{
m_isSimplifiedFileFormat = true;
}
else
{
throw new Exception("Unknown File Mode");
}
m_headerBlockCount = dataReader.ReadByte();
m_lastAllocatedBlock = dataReader.ReadUInt32();
m_snapshotSequenceNumber = dataReader.ReadUInt32();
m_nextFileId = dataReader.ReadUInt16();
m_archiveId = new Guid(dataReader.ReadBytes(16));
m_archiveType = new Guid(dataReader.ReadBytes(16));
int fileCount = dataReader.ReadInt16();
//ToDo: check based on block length
if (fileCount > 64)
{
throw new Exception("Only 64 features are supported per archive");
}
m_files = new ImmutableList <SubFileHeader>(fileCount);
for (int x = 0; x < fileCount; x++)
{
m_files.Add(new SubFileHeader(dataReader, isImmutable: true, isSimplified: m_isSimplifiedFileFormat));
}
m_flags = new ImmutableList <Guid>();
FileHeaderAttributes tag = (FileHeaderAttributes)Encoding7Bit.ReadInt15(dataReader.ReadByte);
while (tag != FileHeaderAttributes.EndOfAttributes)
{
short dataLen;
switch (tag)
{
case FileHeaderAttributes.FileFlags:
dataLen = Encoding7Bit.ReadInt15(dataReader.ReadByte);
while (dataLen > 0)
{
dataLen -= 16;
m_flags.Add(GuidExtensions.ToLittleEndianGuid(dataReader.ReadBytes(16)));
}
break;
case FileHeaderAttributes.UserAttributes:
Guid flag = GuidExtensions.ToLittleEndianGuid(dataReader.ReadBytes(16));
dataLen = Encoding7Bit.ReadInt15(dataReader.ReadByte);
AddUserAttribute(flag, dataReader.ReadBytes(dataLen));
break;
default:
dataLen = Encoding7Bit.ReadInt15(dataReader.ReadByte);
AddUnknownAttribute((byte)tag, dataReader.ReadBytes(dataLen));
break;
}
tag = (FileHeaderAttributes)dataReader.ReadByte();
}
if (!IsFileAllocationTableValid())
{
throw new Exception("File System is invalid");
}
IsReadOnly = true;
}
/// <summary>
/// Reads the 7-bit encoded value from the stream.
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
public static uint Read7BitUInt32(this Stream stream)
{
return(Encoding7Bit.ReadUInt32(stream));
}
private static void TestMethod2()
{
byte[] buffer = new byte[10];
ulong value, result;
int position = 1;
value = 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 2)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 2)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 3)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 3)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127 * 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 4)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 4)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127 * 127 * 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 5)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 5)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127L * 127 * 127 * 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 6)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 6)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127L * 127 * 127 * 127 * 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 7)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 7)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127L * 127 * 127 * 127 * 127 * 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 8)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 8)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127L * 127 * 127 * 127 * 127 * 127 * 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 9)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 9)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = 127L * 127 * 127 * 127 * 127 * 127 * 127 * 127 * 127;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 10)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 10)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
value = ulong.MaxValue;
Encoding7Bit.Write(buffer, ref position, value);
if (position != 10)
{
throw new Exception();
}
position = 1;
result = Encoding7Bit.ReadUInt64(buffer, ref position);
if (position != 10)
{
throw new Exception();
}
if (result != value)
{
throw new Exception();
}
position = 1;
}
public bool IsEndOfStream()
{
int index = m_startIndex;
if (RemainingBytes(index) > 25)
{
return(false);
}
if (RemainingBytes(index) < 0)
{
return(true);
}
byte code;
if (m_nextRunLength > 0)
{
code = m_lastPoint.ExpectedNextCode;
}
else
{
if (RemainingBytes(index) < 1)
{
return(true);
}
code = m_buffer[index++];
}
if ((code & 7) == 5)
{
return(RemainingBytes(index) < 16);
}
if ((code & 7) == 6)
{
return(false);
}
if ((code & 7) == 7)
{
return(RemainingBytes(index) < 1);
}
if ((code & 32) != 0)
{
index += Encoding7Bit.MeasureUInt32(m_buffer, index);
if (RemainingBytes(index) < 0)
{
return(true);
}
}
if ((code & 16) != 0)
{
if (RemainingBytes(index) < 4)
{
return(true);
}
index += 4;
}
if ((code & 7) == 1)
{
if (RemainingBytes(index) < 1)
{
return(true);
}
index += 1;
}
else if ((code & 7) == 2)
{
if (RemainingBytes(index) < 2)
{
return(true);
}
index += 2;
}
else if ((code & 7) == 3)
{
if (RemainingBytes(index) < 3)
{
return(true);
}
index += 3;
}
else if ((code & 7) == 4)
{
if (RemainingBytes(index) < 4)
{
return(true);
}
index += 4;
}
if ((code & 8) != 0)
{
index += Encoding7Bit.MeasureUInt64(m_buffer, index);
if (RemainingBytes(index) < 0)
{
return(true);
}
}
return(false);
}
public unsafe DecompressionExitCode GetMeasurement(out ushort id, out long timestamp, out uint quality, out float value, out byte userCommand)
{
id = 0;
timestamp = 0;
quality = 0;
value = 0;
userCommand = 0;
TryAgain:
int index = m_startIndex;
if (IsEndOfStream())
{
return(DecompressionExitCode.EndOfStreamOccured);
}
byte code;
if (m_nextRunLength > 0)
{
m_nextRunLength--;
code = m_lastPoint.ExpectedNextCode;
}
else
{
code = m_buffer[index++];
m_lastPoint.ExpectedNextCode = code;
m_nextRunLength = code >> 6;
}
PointMetaData point;
if ((code & 7) == 5)
{
point = new PointMetaData();
fixed(byte *signalID = &m_buffer[index])
point.SignalID = *(ushort *)signalID;
index += 2;
point.PointID = m_points.Count;
m_lastPoint.ExpectedNextPointID = m_points.Count;
m_points.Add(point);
m_startIndex = index;
goto TryAgain;
}
if ((code & 7) == 6)
{
throw new NotSupportedException();
}
if ((code & 7) == 7)
{
userCommand = m_buffer[index++];
m_startIndex = index;
return(DecompressionExitCode.CommandRead);
}
if ((code & 32) == 0)
{
point = m_points[m_lastPoint.ExpectedNextPointID];
}
else
{
point = m_points[(int)Encoding7Bit.ReadUInt32(m_buffer, ref index)];
m_lastPoint.ExpectedNextPointID = point.PointID;
}
if ((code & 16) != 0)
{
point.LastQuality = (uint)(m_buffer[index++] | m_buffer[index++] << 8 | m_buffer[index++] << 16 | m_buffer[index++] << 24);
}
if ((code & 7) == 1)
{
point.LastValue ^= m_buffer[index++];
}
else if ((code & 7) == 2)
{
point.LastValue ^= (uint)(m_buffer[index++] | m_buffer[index++] << 8);
}
else if ((code & 7) == 3)
{
point.LastValue ^= (uint)(m_buffer[index++] | m_buffer[index++] << 8 | m_buffer[index++] << 16);
}
else if ((code & 7) == 4)
{
point.LastValue ^= (uint)(m_buffer[index++] | m_buffer[index++] << 8 | m_buffer[index++] << 16 | m_buffer[index++] << 24);
}
if ((code & 8) == 0)
{
timestamp = m_timeBucket;
}
else
{
timestamp = m_timeBucket ^ (long)Encoding7Bit.ReadUInt64(m_buffer, ref index);
m_timeBucket = timestamp;
}
id = point.SignalID;
quality = point.LastQuality;
uint lastValue = point.LastValue;
value = *(float *)&lastValue;
m_lastPoint = point;
m_startIndex = index;
return(DecompressionExitCode.MeasurementRead);
}
public static string Run7Bit32(int thousands, BinaryStream bs, uint b)
{
Stopwatch sw1 = new Stopwatch();
Stopwatch sw2 = new Stopwatch();
for (int x2 = 0; x2 < thousands; x2++)
{
bs.Position = 0;
for (int x = 0; x < 100; x++)
{
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
}
}
sw1.Start();
for (int x2 = 0; x2 < thousands; x2++)
{
bs.Position = 0;
for (int x = 0; x < 100; x++)
{
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
bs.Write7Bit(b);
}
}
sw1.Stop();
sw2.Start();
for (int x2 = 0; x2 < thousands; x2++)
{
bs.Position = 0;
for (int x = 0; x < 100; x++)
{
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
bs.Read7BitUInt32();
}
}
sw2.Stop();
return("7Bit32 " + Encoding7Bit.GetSize(b) + "\t" + thousands * 1000 / sw2.Elapsed.TotalSeconds / 1000000 + "\t" +
thousands * 1000 / sw1.Elapsed.TotalSeconds / 1000000);
}
/// <summary>
/// This will return a byte array of data that can be written to an archive file.
/// </summary>
public byte[] GetBytes()
{
if (!IsFileAllocationTableValid())
{
throw new InvalidOperationException("File Allocation Table is invalid");
}
byte[] dataBytes = new byte[m_blockSize];
MemoryStream stream = new MemoryStream(dataBytes);
BinaryWriter dataWriter = new BinaryWriter(stream);
dataWriter.Write(FileAllocationTableHeaderBytes);
if (BitConverter.IsLittleEndian)
{
dataWriter.Write('L');
}
else
{
dataWriter.Write('B');
}
dataWriter.Write((byte)(BitMath.CountBitsSet((uint)(m_blockSize - 1))));
dataWriter.Write(FileAllocationReadTableVersion);
dataWriter.Write(FileAllocationWriteTableVersion);
dataWriter.Write(FileAllocationHeaderVersion);
dataWriter.Write((byte)(m_isSimplifiedFileFormat ? 2 : 1));
dataWriter.Write(m_headerBlockCount);
dataWriter.Write(m_lastAllocatedBlock);
dataWriter.Write(m_snapshotSequenceNumber);
dataWriter.Write(m_nextFileId);
dataWriter.Write(m_archiveId.ToByteArray());
dataWriter.Write(m_archiveType.ToByteArray());
dataWriter.Write((short)m_files.Count);
foreach (SubFileHeader node in m_files)
{
node.Save(dataWriter);
}
//Metadata Flags
if (m_flags.Count > 0)
{
Encoding7Bit.WriteInt15(dataWriter.Write, (short)FileHeaderAttributes.FileFlags);
Encoding7Bit.WriteInt15(dataWriter.Write, (short)(m_flags.Count * 16));
foreach (var flag in m_flags)
{
dataWriter.Write(GuidExtensions.ToLittleEndianBytes(flag));
}
}
if (m_unknownAttributes != null)
{
foreach (var md in m_unknownAttributes)
{
Encoding7Bit.WriteInt15(dataWriter.Write, md.Key);
Encoding7Bit.WriteInt15(dataWriter.Write, (short)md.Value.Length);
dataWriter.Write(md.Value);
}
}
if (m_userAttributes != null)
{
foreach (var md in m_userAttributes)
{
Encoding7Bit.WriteInt15(dataWriter.Write, (short)FileHeaderAttributes.UserAttributes);
dataWriter.Write(GuidExtensions.ToLittleEndianBytes(md.Key));
Encoding7Bit.WriteInt15(dataWriter.Write, (short)md.Value.Length);
dataWriter.Write(md.Value);
}
}
Encoding7Bit.WriteInt15(dataWriter.Write, (short)FileHeaderAttributes.EndOfAttributes);
Encoding7Bit.WriteInt15(dataWriter.Write, 0);
if (stream.Position + 32 > dataBytes.Length)
{
throw new Exception("the file size exceedes the allowable size.");
}
WriteFooterData(dataBytes);
return(dataBytes);
}
