public Task <IEnumerable <T> > GetFromSequenceNumber <T>(StreamEntityType entityType, ulong sequenceNumberStart = 0, ulong sequenceNumberEnd = ulong.MaxValue) where T : StreamEntityBase
{
var entries = _db.HashGetAll($"{_databaseName}-{entityType}");
if (entries != null)
{
var items = new List <T>();
foreach (var entry in entries)
{
try
{
var item = StreamEntityBase.FromProtoBufByteArray(entry.Value) as T;
if (item.SequenceNumber >= sequenceNumberStart && item.SequenceNumber <= sequenceNumberEnd)
{
items.Add(item);
}
}
catch (Exception ex)
{
}
}
return(Task.FromResult(items.AsEnumerable()));
}
else
{
return(Task.FromResult <IEnumerable <T> >(null));
}
}
public override async Task ProcessStreamItem(StreamEntityBase item)
{
if (_persister != null && item != null)
{
if (_numItemsPerBatch > 0 && _persister.SupportsBatches && _batch == null)
{
_batch = await _persister.CreateBatch(_entityType);
_currentNumItemsInBatch = 0;
}
item.PartitionKey = _partitionKey;
var result = (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update ? _persister.Upsert(item, _batch) : _persister.Delete(item, _batch));
if (_batch != null)
{
_currentNumItemsInBatch++;
if (_currentNumItemsInBatch >= _numItemsPerBatch)
{
var storedLatency = await _batch.Commit();
ReportLatency(storedLatency);
_batch = null;
}
}
else
{
var storedLatency = await result;
ReportLatency(storedLatency);
}
}
DisplayLatency();
}
public override Task ProcessStreamItem(StreamEntityBase item)
{
if (item != null)
{
var lockState = Interlocked.CompareExchange(ref _lockState, 2, 0);
while (lockState == 1)
{
Thread.SpinWait(10);
lockState = Interlocked.CompareExchange(ref _lockState, 2, 0);
}
item.PartitionKey = _partitionKey;
var pendingItems = PendingItems;
lock (this)
{
pendingItems.Add(item);
}
if (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update)
{
Interlocked.Increment(ref _numUpserts);
}
else
{
Interlocked.Increment(ref _numDeletes);
}
Interlocked.Exchange(ref _lockState, 0);
}
return(Task.CompletedTask);
}
public Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch tx = null)
{
return(Task.FromResult(new StoredLatency()
{
NumItems = 0, Time = 0.0
}));
}
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
await _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id));
return(new StoredLatency()
{
NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item)
});
}
public void AddItem(StreamEntityBase item, Task task)
{
if (item != null && task != null)
{
lock (this)
{
Items.Add(item);
Tasks.Add(task);
}
}
}
public async Task Delete(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
var task = _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id));
if (batch != null)
{
var stx = batch as StreamPersisterBatch;
stx.Tasks.Add(task);
}
else
{
await task;
}
}
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
StoredLatency storedLatency;
var sw = new Stopwatch();
sw.Start();
var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection);
var itemDictionary = item.ToKeyValueDictionary();
if (batch != null)
{
var cosmosDbBatch = batch as CosmosDBStreamPersisterBatch;
cosmosDbBatch.AddItem(itemDictionary);
storedLatency = new StoredLatency()
{
NumItems = 0, Time = 0.0
};
}
else
{
await _client.UpsertDocumentAsync(collectionUri, itemDictionary,
new RequestOptions { PartitionKey = new PartitionKey(item.PartitionKey) });
storedLatency = new StoredLatency()
{
NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item)
};
}
sw.Stop();
lock (this)
{
_numUpserts++;
_timeSpentUpsert += sw.ElapsedMilliseconds;
var now = DateTime.UtcNow;
if (now > (_lastReported + TimeSpan.FromSeconds(10)))
{
if (_numUpserts > 0)
{
Console.WriteLine($"CosmosDB Persister; Num upserts {_numUpserts / 10} / sec, Avg time per call {_timeSpentUpsert / _numUpserts} ms.");
}
_numUpserts = 0;
_timeSpentUpsert = 0;
_lastReported = now;
}
}
return(storedLatency);
}
public async Task Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection);
try
{
var task = _client.UpsertDocumentAsync(collectionUri, item);
if (batch != null)
{
var stx = batch as StreamPersisterBatch;
stx.Tasks.Add(task);
}
else
{
await task;
}
}
catch (DocumentClientException e)
{
if (e.StatusCode == System.Net.HttpStatusCode.NotFound)
{
try
{
var task = _client.CreateDocumentAsync(collectionUri, item);
if (batch != null)
{
var stx = batch as StreamPersisterBatch;
stx.Tasks.Add(task);
}
else
{
await task;
}
}
catch (Exception ex)
{
}
}
else
{
throw;
}
}
}
public Task <T> GetById <T>(StreamEntityType entityType, string id) where T : StreamEntityBase
{
var data = _db.HashGet($"{_databaseName}-{entityType}", id);
if (data.HasValue)
{
try
{
return(Task.FromResult(StreamEntityBase.FromProtoBufByteArray(data) as T));
}
catch (Exception ex)
{
return(Task.FromResult <T>(null));
}
}
else
{
return(Task.FromResult <T>(null));
}
}
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
double storedLatency = 0.0;
if (batch != null)
{
RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch;
stx.AddItem(item, stx.Batch.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id));
storedLatency = 0.0;
}
else
{
await _db.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id);
storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item);
}
return(new StoredLatency {
NumItems = (batch == null ? 1 : 0), Time = storedLatency
});
}
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
double storedLatency = 0.0;
var data = item.ToProtoBufByteArray();
if (batch != null)
{
RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch;
stx.AddItem(item, stx.Batch.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data));
storedLatency = 0.0;
}
else
{
await _db.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data);
storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item);
}
Interlocked.Increment(ref _numUpserts);
return(new StoredLatency {
NumItems = (batch == null ? 1 : 0), Time = storedLatency
});
}
public Task ProcessStreamItem(StreamEntityBase streamItem)
{
switch (streamItem.EntityType)
{
case StreamEntityType.Command:
return(ProcessStreamItem(streamItem as StreamCommand));
case StreamEntityType.Market:
return(ProcessStreamItem(streamItem as StreamMarket));
case StreamEntityType.Submarket:
return(ProcessStreamItem(streamItem as StreamSubmarket));
case StreamEntityType.Instrument:
return(ProcessStreamItem(streamItem as StreamInstrument));
case StreamEntityType.Portfolio:
return(ProcessStreamItem(streamItem as StreamPortfolio));
case StreamEntityType.Position:
return(ProcessStreamItem(streamItem as StreamPosition));
case StreamEntityType.Order:
return(ProcessStreamItem(streamItem as StreamOrder));
case StreamEntityType.Rule:
return(ProcessStreamItem(streamItem as StreamRule));
case StreamEntityType.Price:
return(ProcessStreamItem(streamItem as StreamPrice));
case StreamEntityType.Trade:
return(ProcessStreamItem(streamItem as StreamTrade));
default:
return(Task.CompletedTask);
}
}
public Task <IEnumerable <T> > GetAll <T>(StreamEntityType entityType) where T : StreamEntityBase
{
var entries = _db.HashGetAll($"{_databaseName}-{entityType}");
if (entries != null)
{
var items = new List <T>();
foreach (var entry in entries)
{
try
{
items.Add(StreamEntityBase.FromProtoBufByteArray(entry.Value) as T);
}
catch (Exception ex)
{
}
}
return(Task.FromResult(items.AsEnumerable()));
}
else
{
return(Task.FromResult <IEnumerable <T> >(null));
}
}
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch tx = null)
{
if (item.EntityType == StreamEntityType.Market)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertMarket(item as StreamMarket);
}
}
else if (item.EntityType == StreamEntityType.Submarket)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertSubmarket(item as StreamSubmarket);
}
}
else if (item.EntityType == StreamEntityType.Instrument)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertInstrument(item as StreamInstrument);
}
}
else if (item.EntityType == StreamEntityType.Portfolio)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertPortfolio(item as StreamPortfolio);
}
}
else if (item.EntityType == StreamEntityType.Position)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertPosition(item as StreamPosition);
}
}
else if (item.EntityType == StreamEntityType.Rule)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertRule(item as StreamRule);
}
}
else if (item.EntityType == StreamEntityType.Price)
{
if (tx == null)
{
await UpsertPrice(item as StreamPrice);
}
else
{
UpsertPriceBatch(item as StreamPrice, tx);
}
}
else
{
Debugger.Break();
}
if (tx == null)
{
return(new StoredLatency()
{
NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item)
});
}
else
{
return(new StoredLatency()
{
NumItems = 0, Time = 0.0
});
}
}
public abstract Task ProcessStreamItem(StreamEntityBase item);
public Task ProcessStreamItem(StreamEntityBase streamItem)
{
return(_streamEntityPersisters[(int)streamItem.EntityType] != null ?
_streamEntityPersisters[(int)streamItem.EntityType].ProcessStreamItem(streamItem) :
Task.CompletedTask);
}
static public double GetStoredLatency(StreamEntityBase item)
{
return((DateTime.UtcNow - item.Date).TotalMilliseconds);
}
public Task ProcessStreamItem(StreamEntityBase streamItem)
{
int partitionKey = (_numPartitions > 0 ? Math.Max(streamItem.GetPartitionKey(_numPartitions), 0) : 0);
return(_partitions[partitionKey].ProcessStreamItem(streamItem));
}
