// Values returns a copy of all values, deduped and sorted, for the given key. public ClockValues Values(ulong sid) { ClockValues e = null, snap = null; lock (lockthis) { if (store.ContainsKey(sid)) { e = store[sid]; } if (snapshot != null && snapshot.store.ContainsKey(sid)) { snap = snapshot.store[sid]; } } if (e == null && snap == null) { return(null); } else if (e == null) { return(new ClockValues(snap)); } else if (snap == null) { return(new ClockValues(e)); } else { ClockValues result = new ClockValues(e); result.AddRange(snap); return(result); } }
public void DeleteRange(List <ulong> sids, long min, long max) { lock (lockthis) { foreach (ulong sid in sids) { if (!store.ContainsKey(sid)) { continue; } ClockValues e = store[sid]; long origSize = e.Size; if (min == Constants.MinTime && max == Constants.MaxTime) { selfsize -= origSize; store.Remove(sid); continue; } e.Exclude(min, max); if (e.Count() == 0) { store.Remove(sid); selfsize -= origSize; } } Interlocked.Add(ref stats.MemSizeBytes, Size); } }
public Dictionary <ulong, ClockValues> Read(List <ulong> sids) { Dictionary <ulong, ClockValues> result = new Dictionary <ulong, ClockValues>(); lock (lockthis) { result = fileStore.Read(sids); foreach (ulong sid in sids) { ClockValues cachedsidvalues = cache.Values(sid); if (cachedsidvalues != null && cachedsidvalues.Count > 0) { if (!result.ContainsKey(sid)) { result.Add(sid, cachedsidvalues); } else { result[sid].AddRange(cachedsidvalues); } } } } return(result); }
// Write writes the set of values for the key to the cache. This function is goroutine-safe. // It returns an error if the cache will exceed its max size by adding the new values. public string Write(ulong sid, ClockValues values) { long addedSize = values.Size; // Enough room in the cache? long limit = MaxSize; long n = Size + addedSize; if (limit > 0 && n > limit) { Interlocked.Increment(ref stats.WriteErr); return(Constants.ErrCacheMemorySizeLimitExceeded); } lock (lockthis) { if (!store.ContainsKey(sid)) { store.Add(sid, values); } else { addedSize -= store[sid].Size; store[sid].AddRange(values); addedSize += store[sid].Size; } }//lock store. selfsize += addedSize; updateMemSize(addedSize); Interlocked.Increment(ref stats.WriteOK); return(null); }
public static ClockValues Decode2(byte[] bytes, int startindex) { byte datatype = bytes[startindex]; startindex++; int index; index = startindex + Varint.Read(bytes, startindex, out int tsLen);//时标段; TimeDecoder tdec = (TimeDecoder)DecoderFactory.Get(DataTypeEnum.DateTime); tdec.SetBytes(bytes, index, tsLen); index = index + tsLen; index = index + Varint.Read(bytes, index, out int vsLen);//数值段; IDecoder vdec = DecoderFactory.Get(datatype); vdec.SetBytes(bytes, index, vsLen); index = index + vsLen; index = index + Varint.Read(bytes, index, out int qsLen);//质量段; IntegerDecoder qdec = (IntegerDecoder)DecoderFactory.Get(DataTypeEnum.Integer); qdec.SetBytes(bytes, index, qsLen); ClockValues result = new ClockValues(); if (datatype == DataTypeEnum.Double) { FloatDecoder decoder = (FloatDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } else if (datatype == DataTypeEnum.Boolean) { BooleanDecoder decoder = (BooleanDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } else if (datatype == DataTypeEnum.Integer) { IntegerDecoder decoder = (IntegerDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } else if (datatype == DataTypeEnum.String) { StringDecoder decoder = (StringDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } DecoderFactory.Put(DataTypeEnum.DateTime, tdec); DecoderFactory.Put(DataTypeEnum.Integer, qdec); DecoderFactory.Put(datatype, vdec); return(result); }
public Dictionary <ulong, ClockValues> Read(List <ulong> sids, long start, long end) { Dictionary <ulong, ClockValues> result = new Dictionary <ulong, ClockValues>(); lock (lockthis) { result = fileStore.Read(sids, start, end); foreach (ulong sid in sids) { ClockValues cachedsidvalues = cache.Values(sid); if (cachedsidvalues != null && cachedsidvalues.Count > 0) { foreach (IClockValue cv in cachedsidvalues) { if (cv.Clock >= start && cv.Clock <= end) { if (!result.ContainsKey(sid)) { result.Add(sid, new ClockValues()); } result[sid].Add(cv); } } } } } return(result); }
public string Write(ulong id, ClockValues cvs) { Dictionary <ulong, ClockValues> dic = new Dictionary <ulong, ClockValues> { [id] = cvs }; return(WriteMulti(dic)); }
public int MarshalSize() { if (sz > 0 || Values.Count == 0) { return(sz); } int encLen = 5 * Values.Count;// // Type (1), and Count (4) for each sid // determine required length foreach (KeyValuePair <ulong, ClockValues> pair in Values) { ClockValues v = pair.Value; encLen += 8;//sid(8) int count = v.Count; if (count == 0) { return(0); } encLen += 8 * count;//timestamps(8) switch (v[0].DataType) { case DataTypeEnum.Double: case DataTypeEnum.Integer: encLen += 12 * count; break; case DataTypeEnum.Boolean: encLen += 5 * count; break; case DataTypeEnum.String: foreach (ClockString vv in v) { encLen += 4 + System.Text.Encoding.Default.GetByteCount(vv.Value) + 4; } break; default: return(0); } } sz = encLen; return(sz); }
public static ClockValues Decode(byte[] bytes, int startindex, long start, long end) { Span <byte> span = new Span <byte>(bytes); byte datatype = bytes[startindex]; span = span.Slice(1); //解析时标段; int nn = Varint.Read(span, out int tsLen); Span <byte> ts_src_span = span.Slice(nn, tsLen); ulong[] ts = ArrayPool <ulong> .Shared.Rent(Constants.DefaultMaxPointsPerBlock); Span <ulong> ts_to_span = new Span <ulong>(ts, 0, Constants.DefaultMaxPointsPerBlock); BatchTimeStamp tdec = (BatchTimeStamp)CoderFactory.Get(DataTypeEnum.DateTime); (int tscount, string tserror) = tdec.DecodeAll(ts_src_span, ts_to_span); CoderFactory.Put(DataTypeEnum.DateTime, tdec); if (tscount == 0 || tserror != null) { ArrayPool <ulong> .Shared.Return(ts); return(null); } span = span.Slice(nn + tsLen); //读取数值段. nn = Varint.Read(span, out int vsLen); Span <byte> vs_src_span = span.Slice(nn, vsLen); span = span.Slice(nn + vsLen); //解析质量段. nn = Varint.Read(span, out int qsLen); Span <byte> qs_src_span = span.Slice(nn, qsLen); BatchInt64 qdec = (BatchInt64)CoderFactory.Get(DataTypeEnum.Integer); long[] qs = ArrayPool <long> .Shared.Rent(tscount); Span <long> qs_to_span = new Span <long>(qs, 0, tscount); (int qscount, string qserror) = qdec.DecodeAll(qs_src_span, qs_to_span); CoderFactory.Put(DataTypeEnum.Integer, qdec); if (qscount != tscount || qserror != null) { ArrayPool <long> .Shared.Return(qs); return(null); } //解析数据段. ClockValues result = null; if (datatype == DataTypeEnum.Double) { BatchDouble decoder = (BatchDouble)CoderFactory.Get(datatype); double[] vs = ArrayPool <double> .Shared.Rent(tscount + 1); Span <double> vs_to_span = new Span <double>(vs, 0, tscount + 1); (int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span); CoderFactory.Put(datatype, decoder); if (vscount >= tscount && vserror == null) { result = new ClockValues(tscount); for (int i = 0; i < tscount; i++) { long clock = (long)ts_to_span[i]; if (clock > end) { break; } if (clock >= start) { result.Append(new ClockDouble(clock, vs_to_span[i], (int)qs_to_span[i])); } } } ArrayPool <double> .Shared.Return(vs); } else if (datatype == DataTypeEnum.Boolean) { BatchBoolean decoder = (BatchBoolean)CoderFactory.Get(datatype); bool[] vs = ArrayPool <bool> .Shared.Rent(tscount); Span <bool> vs_to_span = new Span <bool>(vs, 0, tscount); (int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span); CoderFactory.Put(datatype, decoder); if (vscount == tscount && vserror == null) { result = new ClockValues(tscount); for (int i = 0; i < tscount; i++) { long clock = (long)ts_to_span[i]; if (clock > end) { break; } if (clock >= start) { result.Append(new ClockBoolean(clock, vs_to_span[i], (int)qs_to_span[i])); } } } ArrayPool <bool> .Shared.Return(vs); } else if (datatype == DataTypeEnum.Integer) { BatchInt64 decoder = (BatchInt64)CoderFactory.Get(datatype); long[] vs = ArrayPool <long> .Shared.Rent(tscount); Span <long> vs_to_span = new Span <long>(vs, 0, tscount); (int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span); CoderFactory.Put(datatype, decoder); if (vscount == tscount && vserror == null) { result = new ClockValues(tscount); for (int i = 0; i < tscount; i++) { long clock = (long)ts_to_span[i]; if (clock > end) { break; } if (clock >= start) { result.Append(new ClockInt64(clock, vs_to_span[i], (int)qs_to_span[i])); } } } ArrayPool <long> .Shared.Return(vs); } else if (datatype == DataTypeEnum.String) { BatchString decoder = (BatchString)CoderFactory.Get(datatype); string[] vs = ArrayPool <string> .Shared.Rent(tscount); Span <string> vs_to_span = new Span <string>(vs, 0, tscount); (int vscount, string vserror) = decoder.DecodeAll(vs_src_span, vs_to_span); CoderFactory.Put(datatype, decoder); if (vscount == tscount && vserror == null) { result = new ClockValues(tscount); for (int i = 0; i < tscount; i++) { long clock = (long)ts_to_span[i]; if (clock > end) { break; } if (clock >= start) { result.Append(new ClockString(clock, vs_to_span[i], (int)qs_to_span[i])); } } } ArrayPool <string> .Shared.Return(vs); } return(result); }
public Dictionary <ulong, ClockValues> Read(List <ulong> sids) { try { Interlocked.Increment(ref refs); Dictionary <ulong, ClockValues> result = new Dictionary <ulong, ClockValues>(); foreach (ulong sid in sids) { ClockValues sidvalues = null; lock (lockthis) { List <IndexEntry> blocks = index.Entries(sid); if (blocks == null || blocks.Count == 0) { continue; } List <TimeRange> tombstones = index.TombstoneRange(sid); bool hasTombstone = (tombstones != null && tombstones.Count > 0); foreach (IndexEntry block in blocks) { bool skip = false; if (hasTombstone) { foreach (TimeRange t in tombstones) { // Should we skip this block because it contains points that have been deleted if (t.Min <= block.MinTime && t.Max >= block.MaxTime) { skip = true; break; } } } if (skip) { continue; } //TODO: Validate checksum ClockValues temp = readBlock(block); if (hasTombstone) { foreach (TimeRange t in tombstones) { temp.Exclude(t.Min, t.Max); } } if (sidvalues == null) { sidvalues = temp; } else { sidvalues.AddRange(temp); } } } if (sidvalues != null && sidvalues.Count > 0) { result.Add(sid, sidvalues); } } return(result); } finally { Interlocked.Decrement(ref refs); } }
// Encode converts the WriteWALEntry into a byte stream using dst if it // is large enough. If dst is too small, the slice will be grown to fit the // encoded entry. public ByteWriter Marshal(out string err) { // The entries values are encode as follows: // // For each key and slice of values, first a 1 byte type for the []Values // slice is written. Following the type, the length and key bytes are written. // Following the key, a 4 byte count followed by each value as a 8 byte time // and N byte value. The value is dependent on the type being encoded. float64, // int64, use 8 bytes, boolean uses 1 byte, and string is similar to the key encoding, // except that string values have a 4-byte length, and keys only use 2 bytes. // // This structure is then repeated for each key an value slices. // // ┌────────────────────────────────────────────────────────────────────┐ // │ WriteWALEntry │ // ├──────┬─────────┬────────┬───────┬─────────┬─────────┬───┬──────┬───┤ // │ Type │ Key Len │ Key │ Count │ Time │ Value │...│ Type │...│ // │1 byte│ 2 bytes │ N bytes│4 bytes│ 8 bytes │ N bytes │ │1 byte│ │ // └──────┴─────────┴────────┴───────┴─────────┴─────────┴───┴──────┴───┘ err = null; int encLen = MarshalSize(); ByteWriter writer = new ByteWriter(encLen); // Finally, encode the entry byte curType = 0x00; foreach (KeyValuePair <ulong, ClockValues> pair in Values) { ClockValues v = pair.Value; curType = v[0].DataType; writer.Write(curType); writer.Write(pair.Key); writer.Write(v.Count); foreach (IClockValue vv in v) { writer.Write(vv.Clock); if (vv.DataType != curType) { err = string.Format("incorrect value found in {0} slice: {1}", curType, vv.OValue); return(null); } switch (vv.DataType) { case DataTypeEnum.Double: writer.Write(((ClockDouble)vv).Value); break; case DataTypeEnum.Integer: writer.Write(((ClockInt64)vv).Value); break; case DataTypeEnum.Boolean: writer.Write(((ClockBoolean)vv).Value); break; case DataTypeEnum.String: writer.Write(((ClockString)vv).Value); break; default: err = string.Format("unsupported value found in {0} slice: {1}", curType, vv.OValue); return(null); } writer.Write(vv.Quality); } } return(writer); }
public string UnmarshalBinary(byte[] b, int startindex, int endindex) { Values.Clear(); int i = startindex; while (i < endindex) { byte typ = b[i]; i++; if (i + 8 > endindex) { return(Constants.ErrWALCorrupt); } ulong sid = BitConverter.ToUInt64(b, i); i += 8; if (i + 4 > endindex) { return(Constants.ErrWALCorrupt); } int nvals = BitConverter.ToInt32(b, i); i += 4; switch (typ) { case DataTypeEnum.Double: if (i + 16 * nvals > endindex) { return(Constants.ErrWALCorrupt); } ClockValues values = new ClockValues(nvals); for (int j = 0; j < nvals; j++) { long un = BitConverter.ToInt64(b, i); i += 8; double v = FloatDecoder.Float64frombits(BitConverter.ToUInt64(b, i)); i += 8; int q = BitConverter.ToInt32(b, i); i += 4; values.Append(new ClockDouble(un, v, q)); } Values[sid] = values; break; case DataTypeEnum.Integer: if (i + 16 * nvals > endindex) { return(Constants.ErrWALCorrupt); } values = new ClockValues(nvals); for (int j = 0; j < nvals; j++) { long un = BitConverter.ToInt64(b, i); i += 8; long v = (long)BitConverter.ToUInt64(b, i); i += 8; int q = BitConverter.ToInt32(b, i); i += 4; values.Append(new ClockInt64(un, v, q)); } Values[sid] = values; break; case DataTypeEnum.Boolean: if (i + 9 * nvals > endindex) { return(Constants.ErrWALCorrupt); } values = new ClockValues(nvals); for (int j = 0; j < nvals; j++) { long un = BitConverter.ToInt64(b, i); i += 8; bool v = b[i] == 1 ? true : false; i += 1; int q = BitConverter.ToInt32(b, i); i += 4; values.Append(new ClockBoolean(un, v, q)); } Values[sid] = values; break; case DataTypeEnum.String: values = new ClockValues(nvals); for (int j = 0; j < nvals; j++) { if (i + 12 > endindex) { return(Constants.ErrWALCorrupt); } long un = BitConverter.ToInt64(b, i); i += 8; int length = BitConverter.ToInt32(b, i); i += 4; if (i + length > endindex) { return(Constants.ErrWALCorrupt); } string v = System.Text.Encoding.Default.GetString(b, i, length); i += length; int q = BitConverter.ToInt32(b, i); i += 4; values.Append(new ClockString(un, v, q)); } Values[sid] = values; break; default: return(string.Format("unsupported value type: {0}", typ)); } } return(null); }
public ClockValues(ClockValues src) { list = new List <IClockValue>(src.list); }
public void AddRange(ClockValues b) { if (b != null && b.Count > 0) { if (list.Count == 0) { list.AddRange(b); } else { int acount = list.Count, bcount = b.Count; if (list[acount - 1].Clock < b[0].Clock) { list.AddRange(b); } else if (list[0].Clock > b[bcount - 1].Clock) { list.InsertRange(0, b); } else { IClockValue[] array = ArrayPool <IClockValue> .Shared.Rent(acount + bcount); Span <IClockValue> span = array.AsSpan(); int i = 0, j = 0, k = 0; while (i < acount && j < bcount) { if (list[i].Clock < b[j].Clock) { span[k++] = list[i++]; } else if (list[i].Clock == b[j].Clock) { i++; } else { span[k++] = b[j++]; } } while (i < acount) { span[k++] = list[i++]; } while (j < bcount) { span[k++] = b[j++]; } if (k > acount) { for (int l = 0; l < acount; l++) { list[l] = span[l]; } list.AddRange(array.Skip(acount).Take(k - acount)); }//将array的内容复制给list. ArrayPool <IClockValue> .Shared.Return(array); } } } }