public Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch tx = null)
{
return(Task.FromResult(new StoredLatency()
{
NumItems = 0, Time = 0.0
}));
}
public override async Task ProcessStreamItem(StreamEntityBase item)
{
if (_persister != null && item != null)
{
if (_numItemsPerBatch > 0 && _persister.SupportsBatches && _batch == null)
{
_batch = await _persister.CreateBatch(_entityType);
_currentNumItemsInBatch = 0;
}
item.PartitionKey = _partitionKey;
var result = (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update ? _persister.Upsert(item, _batch) : _persister.Delete(item, _batch));
if (_batch != null)
{
_currentNumItemsInBatch++;
if (_currentNumItemsInBatch >= _numItemsPerBatch)
{
var storedLatency = await _batch.Commit();
ReportLatency(storedLatency);
_batch = null;
}
}
else
{
var storedLatency = await result;
ReportLatency(storedLatency);
}
}
DisplayLatency();
}
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
await _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id));
return(new StoredLatency()
{
NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item)
});
}
public override async Task ProcessPendingItems(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
bool anyDataProcessed = false;
SwitchDictionaries();
Interlocked.Increment(ref _numSwitchDirectories);
var processItems = ProcessItems;
while (processItems.Any())
{
int storedItems = 0;
IStreamPersisterBatch batch = null;
if (_persister != null)
{
foreach (var item in processItems)
{
if (_persister.SupportsBatches && batch == null)
{
batch = await _persister.CreateBatch(_entityType);
}
StoredLatency storedLatency;
if (item.Operation == StreamOperation.Insert || item.Operation == StreamOperation.Update)
{
storedLatency = await _persister.Upsert(item, batch);
}
else
{
storedLatency = await _persister.Delete(item, batch);
}
ReportLatency(storedLatency);
storedItems++;
anyDataProcessed = true;
Interlocked.Increment(ref _numProcessedItems);
}
}
else
{
Interlocked.Add(ref _numProcessedItems, processItems.Count);
}
if (batch != null)
{
var storedLatency = await batch.Commit();
ReportLatency(storedLatency);
}
processItems.Clear();
}
DisplayLatency();
await Task.Delay(anyDataProcessed?_holdOffBusy : _holdOffIdle);
}
}
private void UpsertPriceBatch(StreamPrice price, IStreamPersisterBatch tx = null)
{
var batchPersister = tx as SqlBatchStreamPersister;
var IntrumentTmp = price.Id.Split(':')[1];
var InstrumentIdSplit = IntrumentTmp.Split('-');
var MarketId = int.Parse(InstrumentIdSplit[0]);
var SubmarketId = int.Parse(InstrumentIdSplit[1]);
var Id = int.Parse(InstrumentIdSplit[2]);
batchPersister.AddRow((MarketId * 10 + SubmarketId) * 1000000 + Id, price.PriceLatest, price.PriceDate, price.Date, price.SequenceNumber);
}
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
StoredLatency storedLatency;
var sw = new Stopwatch();
sw.Start();
var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection);
var itemDictionary = item.ToKeyValueDictionary();
if (batch != null)
{
var cosmosDbBatch = batch as CosmosDBStreamPersisterBatch;
cosmosDbBatch.AddItem(itemDictionary);
storedLatency = new StoredLatency()
{
NumItems = 0, Time = 0.0
};
}
else
{
await _client.UpsertDocumentAsync(collectionUri, itemDictionary,
new RequestOptions { PartitionKey = new PartitionKey(item.PartitionKey) });
storedLatency = new StoredLatency()
{
NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item)
};
}
sw.Stop();
lock (this)
{
_numUpserts++;
_timeSpentUpsert += sw.ElapsedMilliseconds;
var now = DateTime.UtcNow;
if (now > (_lastReported + TimeSpan.FromSeconds(10)))
{
if (_numUpserts > 0)
{
Console.WriteLine($"CosmosDB Persister; Num upserts {_numUpserts / 10} / sec, Avg time per call {_timeSpentUpsert / _numUpserts} ms.");
}
_numUpserts = 0;
_timeSpentUpsert = 0;
_lastReported = now;
}
}
return(storedLatency);
}
public async Task Delete(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
var task = _client.DeleteDocumentAsync(UriFactory.CreateDocumentUri(_database, _collection, item.Id));
if (batch != null)
{
var stx = batch as StreamPersisterBatch;
stx.Tasks.Add(task);
}
else
{
await task;
}
}
public async Task Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
var collectionUri = UriFactory.CreateDocumentCollectionUri(_database, _collection);
try
{
var task = _client.UpsertDocumentAsync(collectionUri, item);
if (batch != null)
{
var stx = batch as StreamPersisterBatch;
stx.Tasks.Add(task);
}
else
{
await task;
}
}
catch (DocumentClientException e)
{
if (e.StatusCode == System.Net.HttpStatusCode.NotFound)
{
try
{
var task = _client.CreateDocumentAsync(collectionUri, item);
if (batch != null)
{
var stx = batch as StreamPersisterBatch;
stx.Tasks.Add(task);
}
else
{
await task;
}
}
catch (Exception ex)
{
}
}
else
{
throw;
}
}
}
public async Task <StoredLatency> Delete(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
double storedLatency = 0.0;
if (batch != null)
{
RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch;
stx.AddItem(item, stx.Batch.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id));
storedLatency = 0.0;
}
else
{
await _db.HashDeleteAsync($"{_databaseName}-{item.EntityType}", item.Id);
storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item);
}
return(new StoredLatency {
NumItems = (batch == null ? 1 : 0), Time = storedLatency
});
}
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch batch = null)
{
double storedLatency = 0.0;
var data = item.ToProtoBufByteArray();
if (batch != null)
{
RedisStreamPersisterBatch stx = batch as RedisStreamPersisterBatch;
stx.AddItem(item, stx.Batch.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data));
storedLatency = 0.0;
}
else
{
await _db.HashSetAsync($"{_databaseName}-{item.EntityType}", item.Id, data);
storedLatency = StreamEntityPersisterPartition.GetStoredLatency(item);
}
Interlocked.Increment(ref _numUpserts);
return(new StoredLatency {
NumItems = (batch == null ? 1 : 0), Time = storedLatency
});
}
public override async Task ProcessPendingItems(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
bool anyDataProcessed = false;
SwitchDictionaries();
Interlocked.Increment(ref _numSwitchDirectories);
var processItems = ProcessItems;
while (processItems.Values.Any())
{
int storedItems = 0;
IStreamPersisterBatch batch = null;
while (processItems.Values.Any() && !cancellationToken.IsCancellationRequested)
{
if (_persister != null)
{
if (_persister.SupportsBatches && batch == null)
{
batch = await _persister.CreateBatch(_entityType);
}
var processItemPair = processItems.First();
if (processItems.Remove(processItemPair.Key))
{
StoredLatency storedLatency;
if (processItemPair.Value.DeleteItem != null)
{
storedLatency = await _persister.Delete(processItemPair.Value.DeleteItem, batch);
}
else if (processItemPair.Value.UpsertItem != null)
{
storedLatency = await _persister.Upsert(processItemPair.Value.UpsertItem, batch);
}
else
{
storedLatency = new StoredLatency()
{
NumItems = 0, Time = 0.0
}
};
ReportLatency(storedLatency);
storedItems++;
anyDataProcessed = true;
Interlocked.Increment(ref _numProcessedItems);
}
}
else
{
Interlocked.Add(ref _numProcessedItems, processItems.Count);
processItems.Clear();
}
}
if (batch != null)
{
var storedLatency = await batch.Commit();
ReportLatency(storedLatency);
}
}
DisplayLatency();
await Task.Delay(anyDataProcessed?_holdOffBusy : _holdOffIdle);
}
}
public async Task <StoredLatency> Upsert(StreamEntityBase item, IStreamPersisterBatch tx = null)
{
if (item.EntityType == StreamEntityType.Market)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertMarket(item as StreamMarket);
}
}
else if (item.EntityType == StreamEntityType.Submarket)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertSubmarket(item as StreamSubmarket);
}
}
else if (item.EntityType == StreamEntityType.Instrument)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertInstrument(item as StreamInstrument);
}
}
else if (item.EntityType == StreamEntityType.Portfolio)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertPortfolio(item as StreamPortfolio);
}
}
else if (item.EntityType == StreamEntityType.Position)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertPosition(item as StreamPosition);
}
}
else if (item.EntityType == StreamEntityType.Rule)
{
if (item.Operation == StreamOperation.Insert)
{
await InsertRule(item as StreamRule);
}
}
else if (item.EntityType == StreamEntityType.Price)
{
if (tx == null)
{
await UpsertPrice(item as StreamPrice);
}
else
{
UpsertPriceBatch(item as StreamPrice, tx);
}
}
else
{
Debugger.Break();
}
if (tx == null)
{
return(new StoredLatency()
{
NumItems = 1, Time = StreamEntityPersisterPartition.GetStoredLatency(item)
});
}
else
{
return(new StoredLatency()
{
NumItems = 0, Time = 0.0
});
}
}
