public async Task InsertManyAsync(IEnumerable <IBestBidAsk> olapEntities)
{
var total = olapEntities.Select(BestBidAskEntity.CreateEntity);
var grouping = from e in total
group e by new { e.PartitionKey } into cms
select new { key = cms.Key, val = cms.ToList() };
foreach (var item in grouping)
{
var list = item.val;
do
{
int bufferLen = maxbuffer;
if (list.Count < maxbuffer)
{
bufferLen = list.Count;
}
var buffer = list.Take(bufferLen);
//Console.WriteLine("Inserting {0} records", bufferLen);
await _storage.InsertOrMergeBatchAsync(buffer);
list.RemoveRange(0, bufferLen);
} while (list.Count > 0);
}
}
public async Task <bool> UpdateKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList)
{
if (!keyValueList.Any())
{
return(true);
}
var list = new List <KeyValueEntity>();
foreach (var item in keyValueList)
{
var kv = item as KeyValueEntity;
list.Add(kv ?? new KeyValueEntity(item));
}
try
{
await _tableStorage.InsertOrMergeBatchAsync(list);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(false);
}
return(true);
}
public async Task SaveAsync(IEnumerable <FixQuoteModel> quotes)
{
const int batchSize = 100;
var entities = quotes.Select(q => new FixQuoteEntity(q.AssetPair.ToLowerInvariant(), q.Ask, q.Bid, q.FixingTime, q.TradeTime));
foreach (var batch in entities.Batch(batchSize))
{
await _storage.InsertOrMergeBatchAsync(batch);
}
}
public async Task AddOrUpdateBatchAsync(IEnumerable <LimitOrder> orders)
{
var models = orders.Select(x =>
{
var entity = new LimitOrderEntity(GetPartitionKey(x.AssetPairId), GetRowKey(x.Id));
Mapper.Map(x, entity);
return(entity);
});
await _storage.InsertOrMergeBatchAsync(models);
}
public async Task <bool> UpdateKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList)
{
try
{
await _tableStorage.InsertOrMergeBatchAsync(keyValueList.Cast <KeyValueEntity>());
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(false);
}
return(true);
}
public async Task <IEnumerable <IConnectionUrlHistory> > GetByRepositoryIdAsync(string repositoryId)
{
var list = await _tableStorage.GetDataAsync(repositoryId);
if (!list.Any())
{
var pk = ConnectionUrlHistory.GeneratePartitionKey();
list = await _tableStorage.GetDataAsync(pk, x => x.RepositoryId == repositoryId);
foreach (var item in list)
{
item.PartitionKey = item.RepositoryId;
}
await _tableStorage.InsertOrMergeBatchAsync(list);
}
return(list);
}
public async Task InsertOrMergeBatchAsync(IEnumerable <TEntity> items)
{
await _retryService.RetryAsync(async() => await _impl.InsertOrMergeBatchAsync(items), _onModificationsRetryCount);
}
public async Task AddAsync(IEnumerable <IPriceCandlesConsistencyResult> entities, DateTime checkDate)
{
await _tableStorage.InsertOrMergeBatchAsync(PriceCandlesConsistencyResultEntity.CreateBatch(entities, checkDate));
}
public async Task AddAsync(IEnumerable <IHedgingServiceCheckResult> entities, DateTime checkDate)
{
await _tableStorage.InsertOrMergeBatchAsync(HedgingServiceCheckResultEntity.CreateBatch(entities, checkDate));
}
public Task InsertOrMergeBatchAsync(IEnumerable <TEntity> items) => WrapAsync(() => _impl.InsertOrMergeBatchAsync(items), nameof(InsertOrMergeBatchAsync), items);
public async Task AddAsync(IEnumerable <IBalanceAndTransactionAmountCheckResult> entities, DateTime checkDate)
{
await _tableStorage.InsertOrMergeBatchAsync(BalanceAndTransactionAmountCheckResultEntity.CreateBatch(entities, checkDate));
}
public Task AddAsync(string clientId, Choice[] choices)
{
var entities = choices.Select(x => UserChoiceEntity.Create(clientId, x));
return(_tableStorage.InsertOrMergeBatchAsync(entities));
}
public async Task InsertOrMergeAsync(IReadOnlyDictionary <TimeInterval, IEnumerable <IFeedCandle> > dict, string asset)
{
if (dict == null)
{
throw new ArgumentNullException(nameof(dict));
}
if (string.IsNullOrEmpty(asset))
{
throw new ArgumentNullException(nameof(asset));
}
if (!dict.Any() && dict.Values.All(e => e != null) && dict.Values.Any(e => e.Count() > 0))
{
return;
}
var partitionKey = CandleTableEntity.GeneratePartitionKey(asset);
var rowKeys = new List <string>();
var fields = new HashSet <string>(); // which fields to read from table
// 1. Read all { pkey, rowkey } rows
//
var updateEntities = new List <CandleTableEntity>();
foreach (var interval in dict.Keys)
{
var candles = dict[interval];
if (candles != null && candles.Any())
{
// Inside one interval group all candles by distinct row
var groups = candles.GroupBy(candle => candle.RowKey(interval));
rowKeys.AddRange(groups.Select(g => g.Key));
foreach (var group in groups)
{
// Create entity with candles and add it to list
var e = new CandleTableEntity(partitionKey, group.Key); // group.Key = rowKey
e.MergeCandles(group, interval);
updateEntities.Add(e);
// update field
var dates = group.Select(c => c.DateTime);
fields.UnionWith(CandleTableEntity.GetStoreFields(interval, dates.Min(), dates.Max()));
}
}
}
// ... prepare get query
// ... partitionKey = ? AND (rokey = ? OR rowkey = ? OR rowkey = ? OR ...)
TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>();
string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var rowkeyFilters = rowKeys.Select(rowKey => TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey));
var rowkeyFilter = rowkeyFilters.Aggregate((cond1, cond2) => TableQuery.CombineFilters(cond1, TableOperators.Or, cond2));
query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And, rowkeyFilter);
query.SelectColumns = fields.ToList();
// ... reading rows from azure table
List <CandleTableEntity> origEntities = new List <CandleTableEntity>(1);
await _tableStorage.ScanDataAsync(query, list =>
{
origEntities.AddRange(list);
return(Task.FromResult(0));
});
// 2. Update rows (merge entities)
//
var listToUpdate = new List <CandleTableEntity>();
foreach (var updateEntity in updateEntities)
{
var origEntity = origEntities.Where(e => e.PartitionKey == updateEntity.PartitionKey && e.RowKey == updateEntity.RowKey).FirstOrDefault();
if (origEntity != null)
{
origEntity.MergeCandles(updateEntity.Candles, updateEntity.Interval);
listToUpdate.Add(origEntity);
}
else
{
listToUpdate.Add(updateEntity);
}
}
// 3. Write rows in batch
// ... Only 100 records with the same pKey can be updated in one batch operation
foreach (var collection in listToUpdate.ToPieces(100))
{
await _tableStorage.InsertOrMergeBatchAsync(collection);
}
}
public Task AddAnswersAsync(IEnumerable <IAnswer> answers)
{
return(_tableStorage.InsertOrMergeBatchAsync(answers.Select(AnswerEntity.Create)));
}
public virtual async Task InsertOrMergeBatchAsync(IEnumerable<TD> objects)
{
await _tableStorage.InsertOrMergeBatchAsync(objects.Select(InvokeCreate));
}
public async Task AddAsync(IEnumerable <IMarginEventsAccountStatusCheckResult> entities, DateTime checkDate)
{
await _tableStorage.InsertOrMergeBatchAsync(MarginEventsAccountStatusCheckResultEntity.CreateBatch(entities, checkDate));
}
public Task AddOrMergeMarketData(IEnumerable <IMarketData> data)
{
var entities = data.Select(MarketDataEntity.Create);
return(_tableStorage.InsertOrMergeBatchAsync(entities));
}
public Task InsertOrMergeBatchAsync(IEnumerable <T> items)
{
var cryptoItems = items.Select(Encrypt);
return(_storage.InsertOrMergeBatchAsync(cryptoItems));
}
public async Task InsertManyAsync(IEnumerable <IBoxSize> olapEntities)
{
await _storage.InsertOrMergeBatchAsync(olapEntities.Select(BoxSizeEntity.CreateEntity));
}
