public Task <IEnumerable <T> > GetFromSequenceNumber <T>(StreamEntityType entityType, ulong sequenceNumberStart = 0, ulong sequenceNumberEnd = ulong.MaxValue) where T : StreamEntityBase { var entries = _db.HashGetAll($"{_databaseName}-{entityType}"); if (entries != null) { var items = new List <T>(); foreach (var entry in entries) { try { var item = StreamEntityBase.FromProtoBufByteArray(entry.Value) as T; if (item.SequenceNumber >= sequenceNumberStart && item.SequenceNumber <= sequenceNumberEnd) { items.Add(item); } } catch (Exception ex) { } } return(Task.FromResult(items.AsEnumerable())); } else { return(Task.FromResult <IEnumerable <T> >(null)); } }
public override async Task ProcessStreamItem(StreamEntityBase item) { if (_persister != null && item != null) { if (_numItemsPerBatch > 0 && _persister.SupportsBatches && _batch == null) { _batch = await _persister.CreateBatch(_entityType); _currentNumItemsInBatch = 0; } item.PartitionKey = _partitionKey; var result = (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update ? _persister.Upsert(item, _batch) : _persister.Delete(item, _batch)); if (_batch != null) { _currentNumItemsInBatch++; if (_currentNumItemsInBatch >= _numItemsPerBatch) { var storedLatency = await _batch.Commit(); ReportLatency(storedLatency); _batch = null; } } else { var storedLatency = await result; ReportLatency(storedLatency); } } DisplayLatency(); }
public override Task ProcessStreamItem(StreamEntityBase item) { if (item != null) { var lockState = Interlocked.CompareExchange(ref _lockState, 2, 0); while (lockState == 1) { Thread.SpinWait(10); lockState = Interlocked.CompareExchange(ref _lockState, 2, 0); } item.PartitionKey = _partitionKey; var pendingItems = PendingItems; lock (this) { pendingItems.Add(item); } if (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update) { Interlocked.Increment(ref _numUpserts); } else { Interlocked.Increment(ref _numDeletes); } Interlocked.Exchange(ref _lockState, 0); } return(Task.CompletedTask); }
public Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch tx = null) { return(Task.FromResult(new StoredLatency() { NumItems = 0, Time = 0.0 })); }
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null) { await _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id)); return(new StoredLatency() { NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item) }); }
public void AddItem(StreamEntityBase item, Task task) { if (item != null && task != null) { lock (this) { Items.Add(item); Tasks.Add(task); } } }
public async Task Delete(StreamEntityBase item, IStreamPersisterBatch batch = null) { var task = _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id)); if (batch != null) { var stx = batch as StreamPersisterBatch; stx.Tasks.Add(task); } else { await task; } }
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null) { StoredLatency storedLatency; var sw = new Stopwatch(); sw.Start(); var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection); var itemDictionary = item.ToKeyValueDictionary(); if (batch != null) { var cosmosDbBatch = batch as CosmosDBStreamPersisterBatch; cosmosDbBatch.AddItem(itemDictionary); storedLatency = new StoredLatency() { NumItems = 0, Time = 0.0 }; } else { await _client.UpsertDocumentAsync(collectionUri, itemDictionary, new RequestOptions { PartitionKey = new PartitionKey(item.PartitionKey) }); storedLatency = new StoredLatency() { NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item) }; } sw.Stop(); lock (this) { _numUpserts++; _timeSpentUpsert += sw.ElapsedMilliseconds; var now = DateTime.UtcNow; if (now > (_lastReported + TimeSpan.FromSeconds(10))) { if (_numUpserts > 0) { Console.WriteLine($"CosmosDB Persister; Num upserts {_numUpserts / 10} / sec, Avg time per call {_timeSpentUpsert / _numUpserts} ms."); } _numUpserts = 0; _timeSpentUpsert = 0; _lastReported = now; } } return(storedLatency); }
public async Task Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null) { var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection); try { var task = _client.UpsertDocumentAsync(collectionUri, item); if (batch != null) { var stx = batch as StreamPersisterBatch; stx.Tasks.Add(task); } else { await task; } } catch (DocumentClientException e) { if (e.StatusCode == System.Net.HttpStatusCode.NotFound) { try { var task = _client.CreateDocumentAsync(collectionUri, item); if (batch != null) { var stx = batch as StreamPersisterBatch; stx.Tasks.Add(task); } else { await task; } } catch (Exception ex) { } } else { throw; } } }
public Task <T> GetById <T>(StreamEntityType entityType, string id) where T : StreamEntityBase { var data = _db.HashGet($"{_databaseName}-{entityType}", id); if (data.HasValue) { try { return(Task.FromResult(StreamEntityBase.FromProtoBufByteArray(data) as T)); } catch (Exception ex) { return(Task.FromResult <T>(null)); } } else { return(Task.FromResult <T>(null)); } }
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null) { double storedLatency = 0.0; if (batch != null) { RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch; stx.AddItem(item, stx.Batch.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id)); storedLatency = 0.0; } else { await _db.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id); storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item); } return(new StoredLatency { NumItems = (batch == null ? 1 : 0), Time = storedLatency }); }
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null) { double storedLatency = 0.0; var data = item.ToProtoBufByteArray(); if (batch != null) { RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch; stx.AddItem(item, stx.Batch.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data)); storedLatency = 0.0; } else { await _db.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data); storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item); } Interlocked.Increment(ref _numUpserts); return(new StoredLatency { NumItems = (batch == null ? 1 : 0), Time = storedLatency }); }
public Task ProcessStreamItem(StreamEntityBase streamItem) { switch (streamItem.EntityType) { case StreamEntityType.Command: return(ProcessStreamItem(streamItem as StreamCommand)); case StreamEntityType.Market: return(ProcessStreamItem(streamItem as StreamMarket)); case StreamEntityType.Submarket: return(ProcessStreamItem(streamItem as StreamSubmarket)); case StreamEntityType.Instrument: return(ProcessStreamItem(streamItem as StreamInstrument)); case StreamEntityType.Portfolio: return(ProcessStreamItem(streamItem as StreamPortfolio)); case StreamEntityType.Position: return(ProcessStreamItem(streamItem as StreamPosition)); case StreamEntityType.Order: return(ProcessStreamItem(streamItem as StreamOrder)); case StreamEntityType.Rule: return(ProcessStreamItem(streamItem as StreamRule)); case StreamEntityType.Price: return(ProcessStreamItem(streamItem as StreamPrice)); case StreamEntityType.Trade: return(ProcessStreamItem(streamItem as StreamTrade)); default: return(Task.CompletedTask); } }
public Task <IEnumerable <T> > GetAll <T>(StreamEntityType entityType) where T : StreamEntityBase { var entries = _db.HashGetAll($"{_databaseName}-{entityType}"); if (entries != null) { var items = new List <T>(); foreach (var entry in entries) { try { items.Add(StreamEntityBase.FromProtoBufByteArray(entry.Value) as T); } catch (Exception ex) { } } return(Task.FromResult(items.AsEnumerable())); } else { return(Task.FromResult <IEnumerable <T> >(null)); } }
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch tx = null) { if (item.EntityType == StreamEntityType.Market) { if (item.Operation == StreamOperation.Insert) { await InsertMarket(item as StreamMarket); } } else if (item.EntityType == StreamEntityType.Submarket) { if (item.Operation == StreamOperation.Insert) { await InsertSubmarket(item as StreamSubmarket); } } else if (item.EntityType == StreamEntityType.Instrument) { if (item.Operation == StreamOperation.Insert) { await InsertInstrument(item as StreamInstrument); } } else if (item.EntityType == StreamEntityType.Portfolio) { if (item.Operation == StreamOperation.Insert) { await InsertPortfolio(item as StreamPortfolio); } } else if (item.EntityType == StreamEntityType.Position) { if (item.Operation == StreamOperation.Insert) { await InsertPosition(item as StreamPosition); } } else if (item.EntityType == StreamEntityType.Rule) { if (item.Operation == StreamOperation.Insert) { await InsertRule(item as StreamRule); } } else if (item.EntityType == StreamEntityType.Price) { if (tx == null) { await UpsertPrice(item as StreamPrice); } else { UpsertPriceBatch(item as StreamPrice, tx); } } else { Debugger.Break(); } if (tx == null) { return(new StoredLatency() { NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item) }); } else { return(new StoredLatency() { NumItems = 0, Time = 0.0 }); } }
public abstract Task ProcessStreamItem(StreamEntityBase item);
public Task ProcessStreamItem(StreamEntityBase streamItem) { return(_streamEntityPersisters[(int)streamItem.EntityType] != null ? _streamEntityPersisters[(int)streamItem.EntityType].ProcessStreamItem(streamItem) : Task.CompletedTask); }
static public double GetStoredLatency(StreamEntityBase item) { return((DateTime.UtcNow - item.Date).TotalMilliseconds); }
public Task ProcessStreamItem(StreamEntityBase streamItem) { int partitionKey = (_numPartitions > 0 ? Math.Max(streamItem.GetPartitionKey(_numPartitions), 0) : 0); return(_partitions[partitionKey].ProcessStreamItem(streamItem)); }