private static void WriteTestData(TimeSeriesStorage tss)
{
var start = new DateTimeOffset(2015, 4, 1, 0, 0, 0, TimeSpan.Zero);
var data = new[]
{
new { Key = "Time", At = start, Value = 10 },
new { Key = "Money", At = start, Value = 54 },
new { Key = "Is", At = start, Value = 1029 },
new { Key = "Money", At = start.AddHours(1), Value = 546 },
new { Key = "Is", At = start.AddHours(1), Value = 70 },
new { Key = "Time", At = start.AddHours(1), Value = 19 },
new { Key = "Is", At = start.AddHours(2), Value = 64 },
new { Key = "Money", At = start.AddHours(2), Value = 130 },
new { Key = "Time", At = start.AddHours(2), Value = 50 },
};
var writer = tss.CreateWriter();
foreach (var item in data)
{
writer.Append("Simple", item.Key, item.At, item.Value);
}
writer.Commit();
writer.Dispose();
}
public static TimeSeriesStorage GetStorage()
{
var storage = new TimeSeriesStorage("http://localhost:8080/", "TimeSeriesTest", new RavenConfiguration {
RunInMemory = true
});
using (var writer = storage.CreateWriter())
{
writer.CreateType("Simple", new[] { "Value" });
writer.Commit();
}
return(storage);
}
public void Import()
{
var storage = new TimeSeriesStorage("http://*****:*****@"C:\Data\TimeSeries\WikiData"
}
});
Console.WriteLine("running");
var sp = Stopwatch.StartNew();
ImportWikipedia(storage);
sp.Stop();
Console.WriteLine(sp.Elapsed);
}
public unsafe bool HasPendingRollupFrom(DocumentsOperationContext context, Slice key, DateTime time)
{
var t = TimeSeriesStorage.EnsureMillisecondsPrecision(time);
var table = context.Transaction.InnerTransaction.OpenTable(RollupSchema, TimeSeriesRollupTable);
if (table == null)
{
return(false);
}
if (table.ReadByKey(key, out var tvr) == false)
{
return(false);
}
var existingRollup = Bits.SwapBytes(*(long *)tvr.Read((int)RollupColumns.NextRollup, out _));
return(existingRollup <= t.Ticks);
}
public TimeSeriesStats(TimeSeriesStorage timeSeriesStorage, Transaction tx)
{
_timeSeriesStorage = timeSeriesStorage;
tx.CreateTree(TimeSeriesStatsKey);
}
public static List <SingleResult> GetAggregatedValues(TimeSeriesReader reader, RangeGroup rangeSpec, AggregationMode mode)
{
var aggStates = new TimeSeriesAggregation(mode); // we always will aggregate here by Min, Max, First, Last, Sum, Count, Mean
var results = new List <SingleResult>();
foreach (var it in reader.SegmentsOrValues())
{
if (it.IndividualValues != null)
{
AggregateIndividualItems(it.IndividualValues);
}
else
{
//We might need to close the old aggregation range and start a new one
MaybeMoveToNextRange(it.Segment.Start);
// now we need to see if we can consume the whole segment, or
// if the range it cover needs to be broken up to multiple ranges.
// For example, if the segment covers 3 days, but we have group by 1 hour,
// we still have to deal with the individual values
if (it.Segment.End > rangeSpec.End)
{
AggregateIndividualItems(it.Segment.Values);
}
else
{
var span = it.Segment.Summary.SegmentValues.Span;
aggStates.Segment(span);
}
}
}
if (aggStates.Any)
{
var result = new SingleResult
{
Timestamp = rangeSpec.Start,
Values = new Memory <double>(aggStates.Values.ToArray()),
Status = TimeSeriesValuesSegment.Live,
Type = SingleResultType.RolledUp
// TODO: Tag = ""
};
TimeSeriesStorage.AssertNoNanValue(result);
results.Add(result);
}
return(results);
void MaybeMoveToNextRange(DateTime ts)
{
if (rangeSpec.WithinRange(ts))
{
return;
}
if (aggStates.Any)
{
var result = new SingleResult
{
Timestamp = rangeSpec.Start,
Values = new Memory <double>(aggStates.Values.ToArray()),
Status = TimeSeriesValuesSegment.Live,
Type = SingleResultType.RolledUp
// TODO: Tag = ""
};
TimeSeriesStorage.AssertNoNanValue(result);
results.Add(result);
}
rangeSpec.MoveToNextRange(ts);
aggStates.Init();
}
void AggregateIndividualItems(IEnumerable <SingleResult> items)
{
foreach (var cur in items)
{
if (cur.Status == TimeSeriesValuesSegment.Dead)
{
continue;
}
MaybeMoveToNextRange(cur.Timestamp);
aggStates.Step(cur.Values.Span);
}
}
}
public HandleTimeSeriesReferences(Index index, Dictionary <string, HashSet <CollectionName> > referencedCollections, TimeSeriesStorage timeSeriesStorage, DocumentsStorage documentsStorage, IndexStorage indexStorage, Config.Categories.IndexingConfiguration configuration)
: base(index, referencedCollections, documentsStorage, indexStorage, indexStorage.ReferencesForDocuments, configuration)
{
_timeSeriesStorage = timeSeriesStorage;
}
public void UpdateStats(DocumentsOperationContext context, TimeSeriesSliceHolder slicer, CollectionName collection, TimeSeriesValuesSegment segment, DateTime baseline, int modifiedEntries)
{
long previousCount;
DateTime start, end;
context.DocumentDatabase.Metrics.TimeSeries.PutsPerSec.MarkSingleThreaded(modifiedEntries);
context.DocumentDatabase.Metrics.TimeSeries.BytesPutsPerSec.MarkSingleThreaded(segment.NumberOfBytes);
var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage;
var table = GetOrCreateTable(context.Transaction.InnerTransaction, collection);
using (ReadStats(context, table, slicer, out previousCount, out start, out end, out var name))
{
var liveEntries = segment.NumberOfLiveEntries;
if (liveEntries > 0)
{
HandleLiveSegment();
}
if (liveEntries == 0)
{
if (TryHandleDeadSegment() == false)
{
// this ts was completely deleted
TimeSeriesStorage.RemoveTimeSeriesNameFromMetadata(context, slicer.DocId, slicer.Name);
start = end = default;
}
}
using (table.Allocate(out var tvb))
{
tvb.Add(slicer.StatsKey);
tvb.Add(GetPolicy(slicer));
tvb.Add(Bits.SwapBytes(start.Ticks));
tvb.Add(end);
tvb.Add(previousCount + liveEntries);
tvb.Add(name);
table.Set(tvb);
}
}
void HandleLiveSegment()
{
var lastTimestamp = GetLastLiveTimestamp(context, segment, baseline);
if (lastTimestamp > end)
{
end = lastTimestamp; // found later end
}
else
{
var reader = tss.GetReader(context, slicer.DocId, slicer.Name, start, DateTime.MaxValue);
var last = reader.Last();
var lastValueInCurrentSegment = reader.ReadBaselineAsDateTime() == baseline;
end = lastValueInCurrentSegment ? lastTimestamp : last.Timestamp;
}
var first = segment.YieldAllValues(context, baseline, includeDead: false).First().Timestamp;
if (first < start)
{
start = first; // found earlier start
}
if (baseline <= start && first >= start)
{
// start was removed
start = first;
}
}
bool TryHandleDeadSegment()
{
if (previousCount == 0)
{
return(false); // if current and previous are zero it means that this time-series was completely deleted
}
var last = segment.GetLastTimestamp(baseline);
if (baseline <= start && last >= start)
{
// start was removed, need to find the next start
// this segment isn't relevant, so let's get the next one
var next = tss.GetReader(context, slicer.DocId, slicer.Name, start, DateTime.MaxValue).NextSegmentBaseline();
var reader = tss.GetReader(context, slicer.DocId, slicer.Name, next, DateTime.MaxValue);
var first = reader.First();
if (first == default)
{
return(false);
}
start = first.Timestamp;
}
var readerOfLastValue = tss.GetReader(context, slicer.DocId, slicer.Name, start, DateTime.MaxValue);
readerOfLastValue.Last();
var lastValueInCurrentSegment = readerOfLastValue.ReadBaselineAsDateTime() == baseline;
if (baseline <= end && end <= last || lastValueInCurrentSegment)
{
var lastEntry = tss.GetReader(context, slicer.DocId, slicer.Name, start, baseline.AddMilliseconds(-1)).Last();
if (lastEntry == default)
{
return(false);
}
end = lastEntry.Timestamp;
}
return(true);
}
}
private static void ImportWikipedia(TimeSeriesStorage storage)
{
var dir = @"E:\TimeSeries\20150401\Compressed";
var files = Directory.GetFiles(dir, "pagecounts-*.gz", SearchOption.TopDirectoryOnly);
for (var fileIndex = 0; fileIndex < files.Length; fileIndex++)
{
Console.WriteLine("Importing " + fileIndex);
if (fileIndex < 70)
{
continue;
using (var writer = storage.CreateWriter())
{
writer.CreateType("Wiki", new[] { "Views", "Size" });
writer.Commit();
}
}
if (fileIndex > 100)
{
break;
}
var fileName = files[fileIndex];
var path = fileName;
using (var stream = File.OpenRead(path))
using (var uncompressed = new GZipStream(stream, CompressionMode.Decompress))
{
var lines = 0;
var writer = storage.CreateWriter();
try
{
using (var reader = new StreamReader(uncompressed))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (string.IsNullOrEmpty(line))
{
continue;
}
var items = line.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (items.Length < 4)
{
continue;
}
var entryName = items[0] + "/" + WebUtility.UrlDecode(items[1]);
if (entryName.Length > 512)
{
continue;
}
var time = DateTime.ParseExact(fileName.Replace(@"E:\TimeSeries\20150401\Compressed\pagecounts-", "").Replace(".gz", ""), "yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
var views = long.Parse(items[2]);
var size = long.Parse(items[3]);
writer.Append("Wiki", entryName, time, views, size);
if (lines++ % 1000 == 0)
{
writer.Commit();
writer.Dispose();
writer = storage.CreateWriter();
}
}
}
}
finally
{
writer.Commit();
writer.Dispose();
}
}
}
}
public HandleCompareExchangeTimeSeriesReferences(Index index, HashSet <string> collectionsWithCompareExchangeReferences, TimeSeriesStorage timeSeriesStorage, DocumentsStorage documentsStorage, IndexStorage indexStorage, IndexingConfiguration configuration)
: base(index, collectionsWithCompareExchangeReferences, documentsStorage, indexStorage, configuration)
{
_collectionsWithCompareExchangeReferences = collectionsWithCompareExchangeReferences;
_timeSeriesStorage = timeSeriesStorage;
}
public ReplicationTask(TimeSeriesStorage storage)
{
this.storage = storage;
this.storage.TimeSeriesUpdated += SignalUpdate;
cancellation = new CancellationTokenSource();
}
private static void MarkForNextPolicyAfterRollup(DocumentsOperationContext context, Table table, RollupState item, TimeSeriesPolicy policy, TimeSeriesStorage tss,
DateTime rollupEnd)
{
table.DeleteByKey(item.Key);
(long Count, DateTime Start, DateTime End)stats = tss.Stats.GetStats(context, item.DocId, item.Name);
if (stats.End > rollupEnd)
{
// we know that we have values after the current rollup and we need to mark them
var nextRollup = rollupEnd.AddMilliseconds(1);
TimeSeriesReader intoReader = tss.GetReader(context, item.DocId, item.Name, nextRollup, DateTime.MaxValue);
if (intoReader.Init() == false)
{
Debug.Assert(false, "We have values but no segment?");
return;
}
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, intoReader.First().Timestamp);
}
}
}
public MapTimeSeries(Index index, TimeSeriesStorage timeSeriesStorage, IndexStorage indexStorage, MapReduceIndexingContext mapReduceContext, IndexingConfiguration configuration)
: base(index, indexStorage, mapReduceContext, configuration)
{
_timeSeriesStorage = timeSeriesStorage;
}
