private IList <VTTQ> ReadDataCompressed(Timestamp startInclusive, Timestamp endInclusive, int maxValues, long count)
{
PreparedStatement statement = stmtRawFirst;
statement[0] = startInclusive.JavaTicks;
statement[1] = endInclusive.JavaTicks;
var result = new List <VTTQ>(maxValues);
int maxIntervals = maxValues / 3;
int itemsPerInterval = (maxValues < 6) ? (int)count : (int)Math.Ceiling(((double)count) / maxIntervals);
var buffer = new List <VTTQ_D>(itemsPerInterval);
using (var reader = statement.ExecuteReader()) {
while (reader.Read())
{
VTTQ x = ReadVTTQ(reader);
if (!x.V.IsEmpty)
{
buffer.Add(new VTTQ_D(x, x.V.AsDouble()));
}
if (buffer.Count >= itemsPerInterval)
{
FlushBuffer(result, buffer, maxValues);
}
}
}
if (buffer.Count > 0)
{
FlushBuffer(result, buffer, maxValues);
}
return(result);
}
private void FlushBuffer(List <VTTQ> result, List <VTTQ_D> buffer, int maxValues)
{
int N = buffer.Count;
if (N > 3)
{
buffer.Sort(CompareVTTQs);
if (maxValues >= 3)
{
VTTQ a = buffer[0].V;
VTTQ b = buffer[N / 2].V;
VTTQ c = buffer[N - 1].V;
AddByTime(result, a, b, c);
}
else
{
result.Add(buffer[N / 2].V);
}
}
else
{
result.AddRange(buffer.Select(y => y.V));
}
buffer.Clear();
}
public override List <VTTQ> ReadData(Timestamp startInclusive, Timestamp endInclusive, int maxValues, BoundingMethod bounding, QualityFilter filter)
{
long N = CountData(startInclusive, endInclusive, filter);
PreparedStatement statement;
switch (bounding)
{
case BoundingMethod.TakeFirstN:
statement = stmtRawFirstN;
break;
case BoundingMethod.TakeLastN:
statement = stmtRawLastN;
break;
case BoundingMethod.CompressToN:
if (N <= maxValues)
{
return(ReadData(startInclusive, endInclusive, maxValues, BoundingMethod.TakeFirstN, filter));
}
else
{
return(ReadDataCompressed(startInclusive, endInclusive, maxValues, N, filter));
}
default:
throw new Exception($"Unknown BoundingMethod: {bounding}");
}
statement[0] = startInclusive.ToDateTime();
statement[1] = endInclusive.ToDateTime();
statement[2] = maxValues;
var filterHelper = QualityFilterHelper.Make(filter);
int initSize = N < maxValues ? (int)N : maxValues;
var res = new List <VTTQ>(initSize);
using (var reader = statement.ExecuteReader()) {
while (reader.Read())
{
VTTQ vttq = ReadVTTQ(reader);
if (filterHelper.Include(vttq.Q))
{
res.Add(vttq);
}
}
}
if (bounding == BoundingMethod.TakeLastN)
{
res.Reverse();
}
return(res);
}
private static ActiveError?VTTQ2Event(VTTQ vttq)
{
AggregatedEvent?evt = vttq.V.Object <AggregatedEvent>();
if (evt == null)
{
return(null);
}
return(Transform(evt));
}
static void AppendAllSame(Timestamp t, int n, List <VTTQ> list)
{
for (int i = 0; i < n; ++i)
{
var vtq = VTTQ.Make(
DataValue.FromDouble(-17.1321),
t + Duration.FromSeconds(10),
t + Duration.FromSeconds(10),
Quality.Bad);
list.Add(vtq);
}
}
static void AppendSpecial(Timestamp t, List <VTTQ> list)
{
list.Add(VTTQ.Make(DataValue.FromString(""), t + Duration.FromSeconds(10), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.FromString("A"), t + Duration.FromSeconds(11), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.FromString("Ä"), t + Duration.FromSeconds(18), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.FromString("AB"), t + Duration.FromSeconds(19), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.FromString("ÖÜÄ"), t + Duration.FromSeconds(30), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.FromString(new string('P', 255)), t + Duration.FromSeconds(34), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.FromString(new string('a', 500)), t + Duration.FromSeconds(38), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.FromString(new string('ä', 9000)), t + Duration.FromMilliseconds(55005), t, Quality.Good));
list.Add(VTTQ.Make(DataValue.Empty, t + Duration.FromSeconds(30), t, Quality.Good));
}
static void AppendSemiRegular(Timestamp t, int n, List <VTTQ> list)
{
for (int i = 0; i < n; ++i)
{
var vtq = VTTQ.Make(
DataValue.FromDouble(0.54321 + i),
t + Duration.FromSeconds(i * 60),
t + Duration.FromSeconds(i * 60) + Duration.FromMilliseconds(i),
Quality.Good);
list.Add(vtq);
}
}
static void AppendRegular(Timestamp t, int n, List <VTTQ> list)
{
Random rand = new Random(78412);
for (int i = 0; i < n; ++i)
{
var vtq = VTTQ.Make(
DataValue.FromDouble(17.54321),
t + Duration.FromSeconds(i * 5),
t + Duration.FromSeconds(i * 5) + Duration.FromMilliseconds(2L * rand.Next(int.MinValue, int.MaxValue)),
Quality.Good);
list.Add(vtq);
}
}
static void AppendRandom(Timestamp t, int n, List <VTTQ> list)
{
Random rand = new Random(2808);
for (int i = 0; i < n; ++i)
{
var tt = t + Duration.FromMilliseconds(i * rand.Next(0, 5000000));
var vtq = VTTQ.Make(
DataValue.FromDouble(rand.NextDouble()),
tt,
tt + Duration.FromMilliseconds(-1 * (i)),
Quality.Uncertain);
list.Add(vtq);
}
}
private static void AddByTime(List <VTTQ> result, VTTQ a, VTTQ b, VTTQ c)
{
if (a.T < b.T && a.T < c.T)
{
result.Add(a);
if (b.T < c.T)
{
result.Add(b);
result.Add(c);
}
else
{
result.Add(c);
result.Add(b);
}
}
else if (b.T < a.T && b.T < c.T)
{
result.Add(b);
if (a.T < c.T)
{
result.Add(a);
result.Add(c);
}
else
{
result.Add(c);
result.Add(a);
}
}
else
{
result.Add(c);
if (a.T < b.T)
{
result.Add(a);
result.Add(b);
}
else
{
result.Add(b);
result.Add(a);
}
}
}
private List <VTTQ> ReadDataCompressed(Timestamp startInclusive, Timestamp endInclusive, int maxValues, long count, QualityFilter filter)
{
PreparedStatement statement = stmtRawFirst;
statement[0] = startInclusive.ToDateTime();
statement[1] = endInclusive.ToDateTime();
var result = new List <VTTQ>(maxValues);
int maxIntervals = maxValues / 3;
int itemsPerInterval = (maxValues < 6) ? (int)count : (int)Math.Ceiling(((double)count) / maxIntervals);
var buffer = new List <VTTQ_D>(itemsPerInterval);
var filterHelper = QualityFilterHelper.Make(filter);
using (var reader = statement.ExecuteReader()) {
while (reader.Read())
{
VTTQ x = ReadVTTQ(reader);
if (!x.V.IsEmpty)
{
double?value = x.V.AsDouble();
if (value.HasValue && filterHelper.Include(x.Q))
{
buffer.Add(new VTTQ_D(x, value.Value));
}
}
if (buffer.Count >= itemsPerInterval)
{
FlushBuffer(result, buffer, maxValues);
}
}
}
if (buffer.Count > 0)
{
FlushBuffer(result, buffer, maxValues);
}
return(result);
}
public static void Serialize(BinaryWriter writer, List <VTTQ> vtqs, byte binaryVersion)
{
int N = vtqs.Count;
if (N > Common.MaxListLen)
{
throw new System.Exception($"VTTQ_Serializer: May not serialize more than {Common.MaxListLen} items");
}
writer.Write(binaryVersion);
writer.Write(Code);
writer.Write(N);
if (N == 0)
{
return;
}
long timeBase = vtqs[0].T.JavaTicks;
long diffBase = N == 1 ? 0 : vtqs[1].T.JavaTicks - timeBase;
string valBase = vtqs[0].V.JSON;
timeBase -= diffBase;
writer.Write(timeBase);
writer.Write(diffBase);
writer.Write(valBase);
byte[] codeTable = Common.mCodeTable;
for (int k = 0; k < N; ++k)
{
VTTQ vtq = vtqs[k];
int control = (int)vtq.Q;
long time = vtq.T.JavaTicks;
long timeDBDiff = vtq.T_DB.JavaTicks - time;
string val = vtq.V.JsonOrNull ?? "";
long diff = time - timeBase;
int valLen = val.Length;
int bytesComapctVal = (valLen + 1) / 2;
bool compactStr = true;
bool writeStr = true;
if (val == valBase)
{
control |= 0x04;
writeStr = false;
}
else
{
if (bytesComapctVal > 0xFF)
{
compactStr = false;
control |= 0x08;
}
else
{
for (int i = 0; i < valLen; i++)
{
int c = val[i];
if ((c & 0xFF80) != 0 || codeTable[c] == 0xFF)
{
compactStr = false;
control |= 0x08;
break;
}
}
}
}
if (diff == diffBase)
{
control |= 0x10;
writer.Write((byte)control);
}
else
{
if (diff % 1000L == 0)
{
control |= 0x20;
diff /= 1000L;
}
if (diff <= sbyte.MaxValue && diff >= sbyte.MinValue)
{
// 1 Byte
writer.Write((byte)control);
writer.Write((sbyte)diff);
}
else if (diff <= short.MaxValue && diff >= short.MinValue)
{
control |= 0x40; // 2 Byte
writer.Write((byte)control);
writer.Write((short)diff);
}
else if (diff <= int.MaxValue && diff >= int.MinValue)
{
control |= 0x80; // 4 Byte
writer.Write((byte)control);
writer.Write((int)diff);
}
else
{
control |= 0xC0; // 8 Byte
writer.Write((byte)control);
writer.Write(diff);
}
}
if (writeStr)
{
if (compactStr)
{
writer.Write((byte)bytesComapctVal);
for (int i = 0; i < valLen; i += 2)
{
char c0 = val[i];
int x = (codeTable[c0] << 4);
if (i + 1 < valLen)
{
char c1 = val[i + 1];
x |= codeTable[c1];
}
else
{
x |= 0x0F;
}
writer.Write((byte)x);
}
}
else
{
writer.Write(val);
}
}
long absTimeDiffDB = System.Math.Abs(timeDBDiff);
if (absTimeDiffDB <= 0x3F)
{
int diffControl = (int)absTimeDiffDB;
if (timeDBDiff < 0)
{
diffControl |= 0x40;
}
writer.Write((byte)diffControl);
}
else
{
int diffControl = 0x80;
if (timeDBDiff < 0)
{
diffControl |= 0x40;
}
if (absTimeDiffDB <= byte.MaxValue)
{
// 1 Byte
writer.Write((byte)diffControl);
writer.Write((byte)absTimeDiffDB);
}
else if (absTimeDiffDB <= ushort.MaxValue)
{
diffControl |= 0x10; // 2 Byte
writer.Write((byte)diffControl);
writer.Write((ushort)absTimeDiffDB);
}
else if (absTimeDiffDB <= uint.MaxValue)
{
diffControl |= 0x20; // 4 Byte
writer.Write((byte)diffControl);
writer.Write((uint)absTimeDiffDB);
}
else
{
diffControl |= 0x30; // 8 Byte
writer.Write((byte)diffControl);
writer.Write((long)absTimeDiffDB);
}
}
timeBase = time;
diffBase = diff;
valBase = val;
}
}
internal VTTQ_D(VTTQ x, double d)
{
V = x;
D = d;
}
public static List <VTTQ> Deserialize(BinaryReader reader)
{
int binaryVersion = reader.ReadByte();
if (binaryVersion == 0)
{
throw new IOException("Failed to deserialize VTTQ[]: Version byte is zero");
}
if (binaryVersion > Common.CurrentBinaryVersion)
{
throw new IOException("Failed to deserialize VTTQ[]: Wrong version byte");
}
if (reader.ReadByte() != Code)
{
throw new IOException("Failed to deserialize VTTQ[]: Wrong start byte");
}
int N = reader.ReadInt32();
if (N > Common.MaxListLen)
{
throw new System.Exception($"VTTQ_Serializer: May not deserialize more than {Common.MaxListLen} items");
}
var res = new List <VTTQ>(N);
if (N == 0)
{
return(res);
}
long timeBase = reader.ReadInt64();
long diffBase = reader.ReadInt64();
string valBase = reader.ReadString();
char[] buffer = new char[255];
char[] mapCode2Char = Common.mapCode2Char;
for (int k = 0; k < N; ++k)
{
int control = reader.ReadByte();
Quality q = (Quality)(control & 0x03);
long time = timeBase;
long diff = diffBase;
string val = valBase;
if ((control & 0x10) == 0)
{
long diffFactor = ((control & 0x20) == 0) ? 1 : 1000;
int codeByteCount = (control & 0xC0) >> 6;
if (codeByteCount == 0)
{
diff = diffFactor * reader.ReadSByte();
}
else if (codeByteCount == 1)
{
diff = diffFactor * reader.ReadInt16();
}
else if (codeByteCount == 2)
{
diff = diffFactor * reader.ReadInt32();
}
else if (codeByteCount == 3)
{
diff = diffFactor * reader.ReadInt64();
}
}
time += diff;
if ((control & 0x04) == 0)
{
if ((control & 0x08) == 0)
{
int countBytes = reader.ReadByte();
int j = 0;
for (int i = 0; i < countBytes; i++)
{
int b = reader.ReadByte();
int b0 = (0xF0 & b) >> 4;
int b1 = (0x0F & b);
buffer[j++] = mapCode2Char[b0];
if (b1 == 0x0F)
{
break;
}
buffer[j++] = mapCode2Char[b1];
}
val = new string(buffer, 0, j);
}
else
{
val = reader.ReadString();
}
}
long diffDB = 0;
int diffControl = reader.ReadByte();
if ((diffControl & 0x80) == 0)
{
int abs = diffControl & 0x3F;
diffDB = (diffControl & 0x40) == 0 ? abs : -abs;
}
else
{
long diffFactor = (diffControl & 0x40) == 0 ? 1 : -1;
int codeByteCount = (diffControl & 0x30) >> 4;
if (codeByteCount == 0)
{
diffDB = diffFactor * reader.ReadByte();
}
else if (codeByteCount == 1)
{
diffDB = diffFactor * reader.ReadUInt16();
}
else if (codeByteCount == 2)
{
diffDB = diffFactor * reader.ReadUInt32();
}
else if (codeByteCount == 3)
{
diffDB = diffFactor * reader.ReadInt64();
}
}
res.Add(VTTQ.Make(DataValue.FromJSON(val), Timestamp.FromJavaTicks(time), Timestamp.FromJavaTicks(time + diffDB), q));
timeBase = time;
diffBase = diff;
valBase = val;
}
return(res);
}
private static ActiveError VTTQ2Event(VTTQ vttq)
{
return(Transform(vttq.V.Object <AggregatedEvent>()));
}
private static AggregatedEvent VTTQ2AggregatedEvent(VTTQ vttq)
{
return(vttq.V.Object <AggregatedEvent>());
}
public override List <VTTQ> ReadData(Timestamp startInclusive, Timestamp endInclusive, int maxValues, BoundingMethod bounding, QualityFilter filter)
{
long N = CountData(startInclusive, endInclusive, filter);
//double millis = (endInclusive - startInclusive).TotalMilliseconds + 1;
//double avgInterval_MS = millis / N;
//double requestInterval_MS = 60 * 1000;
//double N_2 = N;
//if (requestInterval_MS > avgInterval_MS) {
// N_2 = N / (requestInterval_MS / avgInterval_MS);
//}
PreparedStatement statement;
switch (bounding)
{
case BoundingMethod.TakeFirstN:
statement = stmtRawFirstN;
break;
case BoundingMethod.TakeLastN:
statement = stmtRawLastN;
break;
case BoundingMethod.CompressToN:
if (N <= maxValues)
{
return(ReadData(startInclusive, endInclusive, maxValues, BoundingMethod.TakeFirstN, filter));
}
else
{
return(ReadDataCompressed(startInclusive, endInclusive, maxValues, N, filter));
}
default:
throw new Exception($"Unknown BoundingMethod: {bounding}");
}
statement[0] = startInclusive.JavaTicks;
statement[1] = endInclusive.JavaTicks;
statement[2] = maxValues;
var filterHelper = QualityFilterHelper.Make(filter);
int initSize = N < maxValues ? (int)N : maxValues;
var res = new List <VTTQ>(initSize);
using (var reader = statement.ExecuteReader()) {
while (reader.Read())
{
VTTQ vttq = ReadVTTQ(reader);
if (filterHelper.Include(vttq.Q))
{
res.Add(vttq);
}
}
}
if (bounding == BoundingMethod.TakeLastN)
{
res.Reverse();
}
return(res);
}