// Values returns a copy of all values, deduped and sorted, for the given key.
public ClockValues Values(ulong sid)
{
ClockValues e = null, snap = null;
lock (lockthis)
{
if (store.ContainsKey(sid))
{
e = store[sid];
}
if (snapshot != null && snapshot.store.ContainsKey(sid))
{
snap = snapshot.store[sid];
}
}
if (e == null && snap == null)
{
return(null);
}
else if (e == null)
{
return(new ClockValues(snap));
}
else if (snap == null)
{
return(new ClockValues(e));
}
else
{
ClockValues result = new ClockValues(e);
result.AddRange(snap);
return(result);
}
}
public void DeleteRange(List <ulong> sids, long min, long max)
{
lock (lockthis)
{
foreach (ulong sid in sids)
{
if (!store.ContainsKey(sid))
{
continue;
}
ClockValues e = store[sid];
long origSize = e.Size;
if (min == Constants.MinTime && max == Constants.MaxTime)
{
selfsize -= origSize;
store.Remove(sid);
continue;
}
e.Exclude(min, max);
if (e.Count() == 0)
{
store.Remove(sid);
selfsize -= origSize;
}
}
Interlocked.Add(ref stats.MemSizeBytes, Size);
}
}
public Dictionary <ulong, ClockValues> Read(List <ulong> sids)
{
Dictionary <ulong, ClockValues> result = new Dictionary <ulong, ClockValues>();
lock (lockthis)
{
result = fileStore.Read(sids);
foreach (ulong sid in sids)
{
ClockValues cachedsidvalues = cache.Values(sid);
if (cachedsidvalues != null && cachedsidvalues.Count > 0)
{
if (!result.ContainsKey(sid))
{
result.Add(sid, cachedsidvalues);
}
else
{
result[sid].AddRange(cachedsidvalues);
}
}
}
}
return(result);
}
// Write writes the set of values for the key to the cache. This function is goroutine-safe.
// It returns an error if the cache will exceed its max size by adding the new values.
public string Write(ulong sid, ClockValues values)
{
long addedSize = values.Size;
// Enough room in the cache?
long limit = MaxSize;
long n = Size + addedSize;
if (limit > 0 && n > limit)
{
Interlocked.Increment(ref stats.WriteErr);
return(Constants.ErrCacheMemorySizeLimitExceeded);
}
lock (lockthis)
{
if (!store.ContainsKey(sid))
{
store.Add(sid, values);
}
else
{
addedSize -= store[sid].Size;
store[sid].AddRange(values);
addedSize += store[sid].Size;
}
}//lock store.
selfsize += addedSize;
updateMemSize(addedSize);
Interlocked.Increment(ref stats.WriteOK);
return(null);
}
public static ClockValues Decode2(byte[] bytes, int startindex)
{
byte datatype = bytes[startindex];
startindex++;
int index;
index = startindex + Varint.Read(bytes, startindex, out int tsLen);//时标段;
TimeDecoder tdec = (TimeDecoder)DecoderFactory.Get(DataTypeEnum.DateTime);
tdec.SetBytes(bytes, index, tsLen);
index = index + tsLen;
index = index + Varint.Read(bytes, index, out int vsLen);//数值段;
IDecoder vdec = DecoderFactory.Get(datatype);
vdec.SetBytes(bytes, index, vsLen);
index = index + vsLen;
index = index + Varint.Read(bytes, index, out int qsLen);//质量段;
IntegerDecoder qdec = (IntegerDecoder)DecoderFactory.Get(DataTypeEnum.Integer);
qdec.SetBytes(bytes, index, qsLen);
ClockValues result = new ClockValues();
if (datatype == DataTypeEnum.Double)
{
FloatDecoder decoder = (FloatDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
else if (datatype == DataTypeEnum.Boolean)
{
BooleanDecoder decoder = (BooleanDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
else if (datatype == DataTypeEnum.Integer)
{
IntegerDecoder decoder = (IntegerDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
else if (datatype == DataTypeEnum.String)
{
StringDecoder decoder = (StringDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
DecoderFactory.Put(DataTypeEnum.DateTime, tdec);
DecoderFactory.Put(DataTypeEnum.Integer, qdec);
DecoderFactory.Put(datatype, vdec);
return(result);
}
public Dictionary <ulong, ClockValues> Read(List <ulong> sids, long start, long end)
{
Dictionary <ulong, ClockValues> result = new Dictionary <ulong, ClockValues>();
lock (lockthis)
{
result = fileStore.Read(sids, start, end);
foreach (ulong sid in sids)
{
ClockValues cachedsidvalues = cache.Values(sid);
if (cachedsidvalues != null && cachedsidvalues.Count > 0)
{
foreach (IClockValue cv in cachedsidvalues)
{
if (cv.Clock >= start && cv.Clock <= end)
{
if (!result.ContainsKey(sid))
{
result.Add(sid, new ClockValues());
}
result[sid].Add(cv);
}
}
}
}
}
return(result);
}
public string Write(ulong id, ClockValues cvs)
{
Dictionary <ulong, ClockValues> dic = new Dictionary <ulong, ClockValues>
{
[id] = cvs
};
return(WriteMulti(dic));
}
public int MarshalSize()
{
if (sz > 0 || Values.Count == 0)
{
return(sz);
}
int encLen = 5 * Values.Count;// // Type (1), and Count (4) for each sid
// determine required length
foreach (KeyValuePair <ulong, ClockValues> pair in Values)
{
ClockValues v = pair.Value;
encLen += 8;//sid(8)
int count = v.Count;
if (count == 0)
{
return(0);
}
encLen += 8 * count;//timestamps(8)
switch (v[0].DataType)
{
case DataTypeEnum.Double:
case DataTypeEnum.Integer:
encLen += 12 * count;
break;
case DataTypeEnum.Boolean:
encLen += 5 * count;
break;
case DataTypeEnum.String:
foreach (ClockString vv in v)
{
encLen += 4 + System.Text.Encoding.Default.GetByteCount(vv.Value) + 4;
}
break;
default:
return(0);
}
}
sz = encLen;
return(sz);
}
public static ClockValues Decode(byte[] bytes, int startindex, long start, long end)
{
Span <byte> span = new Span <byte>(bytes);
byte datatype = bytes[startindex];
span = span.Slice(1);
//解析时标段;
int nn = Varint.Read(span, out int tsLen);
Span <byte> ts_src_span = span.Slice(nn, tsLen);
ulong[] ts = ArrayPool <ulong> .Shared.Rent(Constants.DefaultMaxPointsPerBlock);
Span <ulong> ts_to_span = new Span <ulong>(ts, 0, Constants.DefaultMaxPointsPerBlock);
BatchTimeStamp tdec = (BatchTimeStamp)CoderFactory.Get(DataTypeEnum.DateTime);
(int tscount, string tserror) = tdec.DecodeAll(ts_src_span, ts_to_span);
CoderFactory.Put(DataTypeEnum.DateTime, tdec);
if (tscount == 0 || tserror != null)
{
ArrayPool <ulong> .Shared.Return(ts);
return(null);
}
span = span.Slice(nn + tsLen);
//读取数值段.
nn = Varint.Read(span, out int vsLen);
Span <byte> vs_src_span = span.Slice(nn, vsLen);
span = span.Slice(nn + vsLen);
//解析质量段.
nn = Varint.Read(span, out int qsLen);
Span <byte> qs_src_span = span.Slice(nn, qsLen);
BatchInt64 qdec = (BatchInt64)CoderFactory.Get(DataTypeEnum.Integer);
long[] qs = ArrayPool <long> .Shared.Rent(tscount);
Span <long> qs_to_span = new Span <long>(qs, 0, tscount);
(int qscount, string qserror) = qdec.DecodeAll(qs_src_span, qs_to_span);
CoderFactory.Put(DataTypeEnum.Integer, qdec);
if (qscount != tscount || qserror != null)
{
ArrayPool <long> .Shared.Return(qs);
return(null);
}
//解析数据段.
ClockValues result = null;
if (datatype == DataTypeEnum.Double)
{
BatchDouble decoder = (BatchDouble)CoderFactory.Get(datatype);
double[] vs = ArrayPool <double> .Shared.Rent(tscount + 1);
Span <double> vs_to_span = new Span <double>(vs, 0, tscount + 1);
(int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span);
CoderFactory.Put(datatype, decoder);
if (vscount >= tscount && vserror == null)
{
result = new ClockValues(tscount);
for (int i = 0; i < tscount; i++)
{
long clock = (long)ts_to_span[i];
if (clock > end)
{
break;
}
if (clock >= start)
{
result.Append(new ClockDouble(clock, vs_to_span[i], (int)qs_to_span[i]));
}
}
}
ArrayPool <double> .Shared.Return(vs);
}
else if (datatype == DataTypeEnum.Boolean)
{
BatchBoolean decoder = (BatchBoolean)CoderFactory.Get(datatype);
bool[] vs = ArrayPool <bool> .Shared.Rent(tscount);
Span <bool> vs_to_span = new Span <bool>(vs, 0, tscount);
(int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span);
CoderFactory.Put(datatype, decoder);
if (vscount == tscount && vserror == null)
{
result = new ClockValues(tscount);
for (int i = 0; i < tscount; i++)
{
long clock = (long)ts_to_span[i];
if (clock > end)
{
break;
}
if (clock >= start)
{
result.Append(new ClockBoolean(clock, vs_to_span[i], (int)qs_to_span[i]));
}
}
}
ArrayPool <bool> .Shared.Return(vs);
}
else if (datatype == DataTypeEnum.Integer)
{
BatchInt64 decoder = (BatchInt64)CoderFactory.Get(datatype);
long[] vs = ArrayPool <long> .Shared.Rent(tscount);
Span <long> vs_to_span = new Span <long>(vs, 0, tscount);
(int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span);
CoderFactory.Put(datatype, decoder);
if (vscount == tscount && vserror == null)
{
result = new ClockValues(tscount);
for (int i = 0; i < tscount; i++)
{
long clock = (long)ts_to_span[i];
if (clock > end)
{
break;
}
if (clock >= start)
{
result.Append(new ClockInt64(clock, vs_to_span[i], (int)qs_to_span[i]));
}
}
}
ArrayPool <long> .Shared.Return(vs);
}
else if (datatype == DataTypeEnum.String)
{
BatchString decoder = (BatchString)CoderFactory.Get(datatype);
string[] vs = ArrayPool <string> .Shared.Rent(tscount);
Span <string> vs_to_span = new Span <string>(vs, 0, tscount);
(int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span);
CoderFactory.Put(datatype, decoder);
if (vscount == tscount && vserror == null)
{
result = new ClockValues(tscount);
for (int i = 0; i < tscount; i++)
{
long clock = (long)ts_to_span[i];
if (clock > end)
{
break;
}
if (clock >= start)
{
result.Append(new ClockString(clock, vs_to_span[i], (int)qs_to_span[i]));
}
}
}
ArrayPool <string> .Shared.Return(vs);
}
return(result);
}
public Dictionary <ulong, ClockValues> Read(List <ulong> sids)
{
try
{
Interlocked.Increment(ref refs);
Dictionary <ulong, ClockValues> result = new Dictionary <ulong, ClockValues>();
foreach (ulong sid in sids)
{
ClockValues sidvalues = null;
lock (lockthis)
{
List <IndexEntry> blocks = index.Entries(sid);
if (blocks == null || blocks.Count == 0)
{
continue;
}
List <TimeRange> tombstones = index.TombstoneRange(sid);
bool hasTombstone = (tombstones != null && tombstones.Count > 0);
foreach (IndexEntry block in blocks)
{
bool skip = false;
if (hasTombstone)
{
foreach (TimeRange t in tombstones)
{
// Should we skip this block because it contains points that have been deleted
if (t.Min <= block.MinTime && t.Max >= block.MaxTime)
{
skip = true;
break;
}
}
}
if (skip)
{
continue;
}
//TODO: Validate checksum
ClockValues temp = readBlock(block);
if (hasTombstone)
{
foreach (TimeRange t in tombstones)
{
temp.Exclude(t.Min, t.Max);
}
}
if (sidvalues == null)
{
sidvalues = temp;
}
else
{
sidvalues.AddRange(temp);
}
}
}
if (sidvalues != null && sidvalues.Count > 0)
{
result.Add(sid, sidvalues);
}
}
return(result);
}
finally
{
Interlocked.Decrement(ref refs);
}
}
// Encode converts the WriteWALEntry into a byte stream using dst if it
// is large enough. If dst is too small, the slice will be grown to fit the
// encoded entry.
public ByteWriter Marshal(out string err)
{
// The entries values are encode as follows:
//
// For each key and slice of values, first a 1 byte type for the []Values
// slice is written. Following the type, the length and key bytes are written.
// Following the key, a 4 byte count followed by each value as a 8 byte time
// and N byte value. The value is dependent on the type being encoded. float64,
// int64, use 8 bytes, boolean uses 1 byte, and string is similar to the key encoding,
// except that string values have a 4-byte length, and keys only use 2 bytes.
//
// This structure is then repeated for each key an value slices.
//
// ┌────────────────────────────────────────────────────────────────────┐
// │ WriteWALEntry │
// ├──────┬─────────┬────────┬───────┬─────────┬─────────┬───┬──────┬───┤
// │ Type │ Key Len │ Key │ Count │ Time │ Value │...│ Type │...│
// │1 byte│ 2 bytes │ N bytes│4 bytes│ 8 bytes │ N bytes │ │1 byte│ │
// └──────┴─────────┴────────┴───────┴─────────┴─────────┴───┴──────┴───┘
err = null;
int encLen = MarshalSize();
ByteWriter writer = new ByteWriter(encLen);
// Finally, encode the entry
byte curType = 0x00;
foreach (KeyValuePair <ulong, ClockValues> pair in Values)
{
ClockValues v = pair.Value;
curType = v[0].DataType;
writer.Write(curType);
writer.Write(pair.Key);
writer.Write(v.Count);
foreach (IClockValue vv in v)
{
writer.Write(vv.Clock);
if (vv.DataType != curType)
{
err = string.Format("incorrect value found in {0} slice: {1}", curType, vv.OValue);
return(null);
}
switch (vv.DataType)
{
case DataTypeEnum.Double:
writer.Write(((ClockDouble)vv).Value);
break;
case DataTypeEnum.Integer:
writer.Write(((ClockInt64)vv).Value);
break;
case DataTypeEnum.Boolean:
writer.Write(((ClockBoolean)vv).Value);
break;
case DataTypeEnum.String:
writer.Write(((ClockString)vv).Value);
break;
default:
err = string.Format("unsupported value found in {0} slice: {1}", curType, vv.OValue);
return(null);
}
writer.Write(vv.Quality);
}
}
return(writer);
}
public string UnmarshalBinary(byte[] b, int startindex, int endindex)
{
Values.Clear();
int i = startindex;
while (i < endindex)
{
byte typ = b[i];
i++;
if (i + 8 > endindex)
{
return(Constants.ErrWALCorrupt);
}
ulong sid = BitConverter.ToUInt64(b, i);
i += 8;
if (i + 4 > endindex)
{
return(Constants.ErrWALCorrupt);
}
int nvals = BitConverter.ToInt32(b, i);
i += 4;
switch (typ)
{
case DataTypeEnum.Double:
if (i + 16 * nvals > endindex)
{
return(Constants.ErrWALCorrupt);
}
ClockValues values = new ClockValues(nvals);
for (int j = 0; j < nvals; j++)
{
long un = BitConverter.ToInt64(b, i);
i += 8;
double v = FloatDecoder.Float64frombits(BitConverter.ToUInt64(b, i));
i += 8;
int q = BitConverter.ToInt32(b, i);
i += 4;
values.Append(new ClockDouble(un, v, q));
}
Values[sid] = values;
break;
case DataTypeEnum.Integer:
if (i + 16 * nvals > endindex)
{
return(Constants.ErrWALCorrupt);
}
values = new ClockValues(nvals);
for (int j = 0; j < nvals; j++)
{
long un = BitConverter.ToInt64(b, i);
i += 8;
long v = (long)BitConverter.ToUInt64(b, i);
i += 8;
int q = BitConverter.ToInt32(b, i);
i += 4;
values.Append(new ClockInt64(un, v, q));
}
Values[sid] = values;
break;
case DataTypeEnum.Boolean:
if (i + 9 * nvals > endindex)
{
return(Constants.ErrWALCorrupt);
}
values = new ClockValues(nvals);
for (int j = 0; j < nvals; j++)
{
long un = BitConverter.ToInt64(b, i);
i += 8;
bool v = b[i] == 1 ? true : false;
i += 1;
int q = BitConverter.ToInt32(b, i);
i += 4;
values.Append(new ClockBoolean(un, v, q));
}
Values[sid] = values;
break;
case DataTypeEnum.String:
values = new ClockValues(nvals);
for (int j = 0; j < nvals; j++)
{
if (i + 12 > endindex)
{
return(Constants.ErrWALCorrupt);
}
long un = BitConverter.ToInt64(b, i);
i += 8;
int length = BitConverter.ToInt32(b, i);
i += 4;
if (i + length > endindex)
{
return(Constants.ErrWALCorrupt);
}
string v = System.Text.Encoding.Default.GetString(b, i, length);
i += length;
int q = BitConverter.ToInt32(b, i);
i += 4;
values.Append(new ClockString(un, v, q));
}
Values[sid] = values;
break;
default:
return(string.Format("unsupported value type: {0}", typ));
}
}
return(null);
}
public ClockValues(ClockValues src)
{
list = new List <IClockValue>(src.list);
}
public void AddRange(ClockValues b)
{
if (b != null && b.Count > 0)
{
if (list.Count == 0)
{
list.AddRange(b);
}
else
{
int acount = list.Count, bcount = b.Count;
if (list[acount - 1].Clock < b[0].Clock)
{
list.AddRange(b);
}
else if (list[0].Clock > b[bcount - 1].Clock)
{
list.InsertRange(0, b);
}
else
{
IClockValue[] array = ArrayPool <IClockValue> .Shared.Rent(acount + bcount);
Span <IClockValue> span = array.AsSpan();
int i = 0, j = 0, k = 0;
while (i < acount && j < bcount)
{
if (list[i].Clock < b[j].Clock)
{
span[k++] = list[i++];
}
else if (list[i].Clock == b[j].Clock)
{
i++;
}
else
{
span[k++] = b[j++];
}
}
while (i < acount)
{
span[k++] = list[i++];
}
while (j < bcount)
{
span[k++] = b[j++];
}
if (k > acount)
{
for (int l = 0; l < acount; l++)
{
list[l] = span[l];
}
list.AddRange(array.Skip(acount).Take(k - acount));
}//将array的内容复制给list.
ArrayPool <IClockValue> .Shared.Return(array);
}
}
}
}