public Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch tx = null) { return(Task.FromResult(new StoredLatency() { NumItems = 0, Time = 0.0 })); }
public override async Task ProcessStreamItem(StreamEntityBase item) { if (_persister != null && item != null) { if (_numItemsPerBatch > 0 && _persister.SupportsBatches && _batch == null) { _batch = await _persister.CreateBatch(_entityType); _currentNumItemsInBatch = 0; } item.PartitionKey = _partitionKey; var result = (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update ? _persister.Upsert(item, _batch) : _persister.Delete(item, _batch)); if (_batch != null) { _currentNumItemsInBatch++; if (_currentNumItemsInBatch >= _numItemsPerBatch) { var storedLatency = await _batch.Commit(); ReportLatency(storedLatency); _batch = null; } } else { var storedLatency = await result; ReportLatency(storedLatency); } } DisplayLatency(); }
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null) { await _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id)); return(new StoredLatency() { NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item) }); }
public override async Task ProcessPendingItems(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { bool anyDataProcessed = false; SwitchDictionaries(); Interlocked.Increment(ref _numSwitchDirectories); var processItems = ProcessItems; while (processItems.Any()) { int storedItems = 0; IStreamPersisterBatch batch = null; if (_persister != null) { foreach (var item in processItems) { if (_persister.SupportsBatches && batch == null) { batch = await _persister.CreateBatch(_entityType); } StoredLatency storedLatency; if (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update) { storedLatency = await _persister.Upsert(item, batch); } else { storedLatency = await _persister.Delete(item, batch); } ReportLatency(storedLatency); storedItems++; anyDataProcessed = true; Interlocked.Increment(ref _numProcessedItems); } } else { Interlocked.Add(ref _numProcessedItems, processItems.Count); } if (batch != null) { var storedLatency = await batch.Commit(); ReportLatency(storedLatency); } processItems.Clear(); } DisplayLatency(); await Task.Delay(anyDataProcessed?_holdOffBusy : _holdOffIdle); } }
private void UpsertPriceBatch(StreamPrice price, IStreamPersisterBatch tx = null) { var batchPersister = tx as SqlBatchStreamPersister; var IntrumentTmp = price.Id.Split(':')[1]; var InstrumentIdSplit = IntrumentTmp.Split('-'); var MarketId = int.Parse(InstrumentIdSplit[0]); var SubmarketId = int.Parse(InstrumentIdSplit[1]); var Id = int.Parse(InstrumentIdSplit[2]); batchPersister.AddRow((MarketId * 10 + SubmarketId) * 1000000 + Id, price.PriceLatest, price.PriceDate, price.Date, price.SequenceNumber); }
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null) { StoredLatency storedLatency; var sw = new Stopwatch(); sw.Start(); var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection); var itemDictionary = item.ToKeyValueDictionary(); if (batch != null) { var cosmosDbBatch = batch as CosmosDBStreamPersisterBatch; cosmosDbBatch.AddItem(itemDictionary); storedLatency = new StoredLatency() { NumItems = 0, Time = 0.0 }; } else { await _client.UpsertDocumentAsync(collectionUri, itemDictionary, new RequestOptions { PartitionKey = new PartitionKey(item.PartitionKey) }); storedLatency = new StoredLatency() { NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item) }; } sw.Stop(); lock (this) { _numUpserts++; _timeSpentUpsert += sw.ElapsedMilliseconds; var now = DateTime.UtcNow; if (now > (_lastReported + TimeSpan.FromSeconds(10))) { if (_numUpserts > 0) { Console.WriteLine($"CosmosDB Persister; Num upserts {_numUpserts / 10} / sec, Avg time per call {_timeSpentUpsert / _numUpserts} ms."); } _numUpserts = 0; _timeSpentUpsert = 0; _lastReported = now; } } return(storedLatency); }
public async Task Delete(StreamEntityBase item, IStreamPersisterBatch batch = null) { var task = _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id)); if (batch != null) { var stx = batch as StreamPersisterBatch; stx.Tasks.Add(task); } else { await task; } }
public async Task Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null) { var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection); try { var task = _client.UpsertDocumentAsync(collectionUri, item); if (batch != null) { var stx = batch as StreamPersisterBatch; stx.Tasks.Add(task); } else { await task; } } catch (DocumentClientException e) { if (e.StatusCode == System.Net.HttpStatusCode.NotFound) { try { var task = _client.CreateDocumentAsync(collectionUri, item); if (batch != null) { var stx = batch as StreamPersisterBatch; stx.Tasks.Add(task); } else { await task; } } catch (Exception ex) { } } else { throw; } } }
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null) { double storedLatency = 0.0; if (batch != null) { RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch; stx.AddItem(item, stx.Batch.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id)); storedLatency = 0.0; } else { await _db.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id); storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item); } return(new StoredLatency { NumItems = (batch == null ? 1 : 0), Time = storedLatency }); }
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null) { double storedLatency = 0.0; var data = item.ToProtoBufByteArray(); if (batch != null) { RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch; stx.AddItem(item, stx.Batch.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data)); storedLatency = 0.0; } else { await _db.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data); storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item); } Interlocked.Increment(ref _numUpserts); return(new StoredLatency { NumItems = (batch == null ? 1 : 0), Time = storedLatency }); }
public override async Task ProcessPendingItems(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { bool anyDataProcessed = false; SwitchDictionaries(); Interlocked.Increment(ref _numSwitchDirectories); var processItems = ProcessItems; while (processItems.Values.Any()) { int storedItems = 0; IStreamPersisterBatch batch = null; while (processItems.Values.Any() && !cancellationToken.IsCancellationRequested) { if (_persister != null) { if (_persister.SupportsBatches && batch == null) { batch = await _persister.CreateBatch(_entityType); } var processItemPair = processItems.First(); if (processItems.Remove(processItemPair.Key)) { StoredLatency storedLatency; if (processItemPair.Value.DeleteItem != null) { storedLatency = await _persister.Delete(processItemPair.Value.DeleteItem, batch); } else if (processItemPair.Value.UpsertItem != null) { storedLatency = await _persister.Upsert(processItemPair.Value.UpsertItem, batch); } else { storedLatency = new StoredLatency() { NumItems = 0, Time = 0.0 } }; ReportLatency(storedLatency); storedItems++; anyDataProcessed = true; Interlocked.Increment(ref _numProcessedItems); } } else { Interlocked.Add(ref _numProcessedItems, processItems.Count); processItems.Clear(); } } if (batch != null) { var storedLatency = await batch.Commit(); ReportLatency(storedLatency); } } DisplayLatency(); await Task.Delay(anyDataProcessed?_holdOffBusy : _holdOffIdle); } }
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch tx = null) { if (item.EntityType == StreamEntityType.Market) { if (item.Operation == StreamOperation.Insert) { await InsertMarket(item as StreamMarket); } } else if (item.EntityType == StreamEntityType.Submarket) { if (item.Operation == StreamOperation.Insert) { await InsertSubmarket(item as StreamSubmarket); } } else if (item.EntityType == StreamEntityType.Instrument) { if (item.Operation == StreamOperation.Insert) { await InsertInstrument(item as StreamInstrument); } } else if (item.EntityType == StreamEntityType.Portfolio) { if (item.Operation == StreamOperation.Insert) { await InsertPortfolio(item as StreamPortfolio); } } else if (item.EntityType == StreamEntityType.Position) { if (item.Operation == StreamOperation.Insert) { await InsertPosition(item as StreamPosition); } } else if (item.EntityType == StreamEntityType.Rule) { if (item.Operation == StreamOperation.Insert) { await InsertRule(item as StreamRule); } } else if (item.EntityType == StreamEntityType.Price) { if (tx == null) { await UpsertPrice(item as StreamPrice); } else { UpsertPriceBatch(item as StreamPrice, tx); } } else { Debugger.Break(); } if (tx == null) { return(new StoredLatency() { NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item) }); } else { return(new StoredLatency() { NumItems = 0, Time = 0.0 }); } }