private async Task ApplyUpdates(DiffObject updates)
{
try
{
var sw = new Stopwatch();
sw.Start();
var retries = await Context.PersistenceManager.ApplyUpdates(updates);
sw.Stop();
var batchInfo = new BatchSavedInfo
{
SavedAt = DateTime.UtcNow,
QuantaCount = updates.Quanta.Count,
EffectsCount = updates.EffectsCount,
ElapsedMilliseconds = sw.ElapsedMilliseconds,
Retries = retries
};
_ = Task.Factory.StartNew(() => OnBatchSaved?.Invoke(batchInfo));
}
catch (Exception exc)
{
//we need to cancel all pending updates
cancellationTokenSource.Cancel();
if (Context.AppState.State != ApplicationState.Failed)
{
OnSaveFailed("Error on saving updates.", exc);
}
}
}
private void ShowDiffObject(StreamWriter stream, DiffObject parent, int level)
{
string indent = level == 0 ? string.Empty : "-".PadLeft(level * 4, '-');
ConfigObject mdSource = parent.SourceValue as ConfigObject;
ConfigObject mdTarget = parent.TargetValue as ConfigObject;
if (mdSource != null && mdTarget != null)
{
stream.WriteLine(CreateObjectPresentation(parent, level, indent));
}
else if (mdSource != null)
{
stream.WriteLine(CreateObjectPresentation(parent, level, indent));
}
else if (mdTarget != null)
{
stream.WriteLine(CreateObjectPresentation(parent, level, indent));
}
else
{
stream.WriteLine(CreateDiffPresentation(parent, level, indent));
}
foreach (DiffObject diff in parent.DiffObjects)
{
ShowDiffObject(stream, diff, level + 1);
}
}
/**********************************************************************************/
public async Task Diff(DiffObject diffObject = null)
{
var now = DateTime.Now;
try
{
Logger.Trace("Diffiniarizing...");
if (diffObject == null)
{
diffObject = new DiffObject();
}
diffObject.LastSyncronizationTimestamp = _settings.LastServerTimestamp;
diffObject.ClientTimeStamp = now.ToTimeStamp();
diffObject.Transactions = GetPendingTransactions();
var uri = new Uri(ApiBaseUrl + "/v6/diff/");
var httpClient = OAuthUtility.CreateOAuthClient(ConsumerKey, ConsumerSecret, _accessToken);
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var postJson = JsonConvert.SerializeObject(diffObject);
Logger.Trace("Request: \n\n" + postJson);
var response = await httpClient.PostAsync(uri, new StringContent(postJson, Encoding.UTF8, "application/json"));
var json = await response.Content.ReadAsStringAsync();
Logger.Trace("Response: \n\n" + json);
var diffResponse = JsonConvert.DeserializeObject <DiffRespnseObject>(json);
_settings.LastSyncTime = now;
_settings.LastServerTimestamp = diffResponse.ServerTimestamp;
await SaveChanges(diffResponse);
if (diffResponse.Transactions != null)
{
var updatetedTransactionIds = diffResponse.Transactions.Select(x => x.Id).ToArray();
Upserts.RemoveAll(x => updatetedTransactionIds.Contains(x.Id));
_db.Table <Upsert>().DeleteByKeys(updatetedTransactionIds);
}
}
catch (Exception ex)
{
Logger.LogException(ex, "Exception in (ZenmoneyClient.cs)\\[ZenmoneyClient.GetProfile] ");
throw;
}
}
public async Task <int> ApplyUpdates(DiffObject updates)
{
await saveSnapshotSemaphore.WaitAsync();
try
{
return(await storage.Update(updates));
}
finally
{
saveSnapshotSemaphore.Release();
}
}
public async Task <int> Update(DiffObject update)
{
var constellationUpdate = GetConstellationUpdate(update.StellarInfoData);
var accountUpdates = GetAccountUpdates(update.Accounts.Values.ToList());
var balanceUpdates = GetBalanceUpdates(update.Balances.Values.ToList());
var orderUpdates = GetOrderUpdates(update.Orders.Values.ToList());
var quanta = GetQuantaUpdates(update.Quanta);
using (var session = await client.StartSessionAsync())
{
var result = await session.WithTransactionAsync <bool>(async (s, ct) =>
{
var updateTasks = new List <Task>();
if (constellationUpdate != null)
{
updateTasks.Add(constellationStateCollection.BulkWriteAsync(s, new [] { constellationUpdate }, cancellationToken: ct));
}
if (update.ConstellationSettings != null)
{
updateTasks.Add(settingsCollection.InsertOneAsync(s, update.ConstellationSettings, cancellationToken: ct));
}
if (accountUpdates != null)
{
updateTasks.AddRange(accountsCollection.WriteBatch(s, accountUpdates, ct));
}
if (balanceUpdates != null)
{
updateTasks.AddRange(balancesCollection.WriteBatch(s, balanceUpdates, ct));
}
if (orderUpdates != null)
{
updateTasks.AddRange(ordersCollection.WriteBatch(s, orderUpdates, ct));
}
updateTasks.AddRange(quantaCollection.WriteBatch(s, quanta, ct));
await Task.WhenAll(updateTasks);
return(true);
},
txOptions,
CancellationToken.None);
return(1);
}
}
public Task <int> Update(DiffObject update)
{
UpdateSettings(update.ConstellationSettings);
UpdateStellarData(update.StellarInfoData);
UpdateAccount(update.Accounts.Values.ToList());
GetBalanceUpdates(update.Balances.Values.ToList());
UpdateOrders(update.Orders.Values.ToList());
UpdateQuanta(update.Quanta);
return(Task.FromResult(1));
}
public void OrderbookPerformanceTest(int iterations, bool useNormalDistribution = false)
{
var rnd = new Random();
var testTradeResults = new Dictionary <RequestQuantum, EffectProcessorsContainer>();
for (var i = 1; i < iterations; i++)
{
var price = useNormalDistribution ? rnd.NextNormallyDistributed() + 50 : rnd.NextDouble() * 100;
var accountWrapper = context.AccountStorage.GetAccount(account1.Account.Pubkey);
var trade = new RequestQuantum
{
Apex = i,
RequestEnvelope = new MessageEnvelope
{
Message = new OrderRequest
{
Account = accountWrapper.Account.Id,
RequestId = i,
Amount = rnd.Next(1, 20),
Asset = 1,
Price = Math.Round(price * 10) / 10,
Side = rnd.NextDouble() >= 0.5 ? OrderSide.Buy : OrderSide.Sell,
AccountWrapper = context.AccountStorage.GetAccount(account1.Account.Pubkey)
},
Signatures = new List <Ed25519Signature>()
},
Timestamp = DateTime.UtcNow.Ticks
};
var diffObject = new DiffObject();
var conterOrderEffectsContainer = new EffectProcessorsContainer(context, trade.CreateEnvelope(), diffObject);
testTradeResults.Add(trade, conterOrderEffectsContainer);
}
PerfCounter.MeasureTime(() =>
{
foreach (var trade in testTradeResults)
{
context.Exchange.ExecuteOrder(trade.Value);
}
}, () =>
{
var market = context.Exchange.GetMarket(1);
return($"{iterations} iterations, orderbook size: {market.Bids.Count} bids, {market.Asks.Count} asks, {market.Bids.GetBestPrice().ToString("G3")}/{market.Asks.GetBestPrice().ToString("G3")} spread.");
});
}
public Task <int> Update(DiffObject update)
{
UpdateSettings(update.ConstellationSettings, update.Assets);
UpdateStellarData(update.StellarInfoData);
UpdateAccount(update.Accounts.Values.ToList());
GetBalanceUpdates(update.Balances.Values.ToList());
UpdateOrders(update.Orders.Values.ToList());
UpdateQuanta(update.Quanta.Select(q => q.Quantum).ToList());
UpdateEffects(update.Quanta.SelectMany(v => v.Effects.Values).ToList());
return(Task.FromResult(1));
}
private string CreateObjectPresentation(DiffObject diff, int level, string indent)
{
string presentation = indent + "[" + level.ToString() + "] (" + diff.Path + ") ";
if (diff.SourceValue == null)
{
presentation += "NULL";
}
else
{
presentation += "\"" + diff.SourceValue.ToString() + "\"";
}
presentation += " : ";
if (diff.TargetValue == null)
{
presentation += "NULL";
}
else
{
presentation += "\"" + diff.TargetValue?.ToString() + "\"";
}
return(presentation);
}
private string CreateDiffPresentation(DiffObject diff, int level, string indent)
{
string token = " "; // None
if (diff.DiffKind == DiffKind.Update)
{
token = "*";
}
else if (diff.DiffKind == DiffKind.Insert)
{
token = "+";
}
else if (diff.DiffKind == DiffKind.Delete)
{
token = "-";
}
string presentation = indent + "[" + level.ToString() + "] " + token + " (" + diff.Path + ") ";
if (diff.SourceValue == null)
{
presentation += "NULL";
}
else
{
presentation += "\"" + diff.SourceValue.ToString() + "\"";
}
presentation += " : ";
if (diff.TargetValue == null)
{
presentation += "NULL";
}
else
{
presentation += "\"" + diff.TargetValue?.ToString() + "\"";
}
return(presentation);
}
private void RefreshUpdatesUnlocked()
{
if (Current.Quanta.Count < 1)
{
return;
}
var pendingUpdates = Current;
Current = new DiffObject();
if (awaitedUpdates == null)
{
return;
}
awaitedUpdates.Add(pendingUpdates);
if (Context.IsAlpha)
{
QuantaThrottlingManager.Current.SetBatchQueueLength(awaitedUpdates.Count);
}
else if (awaitedUpdates.Count >= 20 && Context.AppState.State != ApplicationState.Failed)
{
OnSaveFailed($"Delayed updates queue ({awaitedUpdates.Count}) is too long.");
}
}
public EffectProcessorsContainer(ExecutionContext context, MessageEnvelope quantum, DiffObject pendingDiffObject)
: base(context)
{
Envelope = quantum ?? throw new ArgumentNullException(nameof(quantum));
PendingDiffObject = pendingDiffObject ?? throw new ArgumentNullException(nameof(pendingDiffObject));
}
private void ExecuteWithOrderbook(int iterations, bool useNormalDistribution, Action <Action> executor)
{
var rnd = new Random();
var market = Global.Exchange.GetMarket(1);
var testTradeResults = new Dictionary <RequestQuantum, EffectProcessorsContainer>();
for (var i = 1; i < iterations; i++)
{
var price = useNormalDistribution ? rnd.NextNormallyDistributed() + 50 : rnd.NextDouble() * 100;
var request = new OrderRequest
{
RequestId = i,
Amount = rnd.Next(1, 20),
Asset = 1,
Price = Math.Round(price * 27) / 13
};
if (rnd.NextDouble() >= 0.5)
{
request.Account = account1.Id;
request.AccountWrapper = account1;
request.Side = OrderSide.Buy;
}
else
{
request.Account = account2.Id;
request.AccountWrapper = account2;
request.Side = OrderSide.Sell;
}
var trade = new RequestQuantum
{
Apex = i,
RequestEnvelope = new MessageEnvelope
{
Message = request,
Signatures = new List <Ed25519Signature>()
},
Timestamp = DateTime.UtcNow.Ticks
};
var diffObject = new DiffObject();
var conterOrderEffectsContainer = new EffectProcessorsContainer(trade.CreateEnvelope(), diffObject);
testTradeResults.Add(trade, conterOrderEffectsContainer);
}
var xlmStartBalance = account1.Account.GetBalance(0).Amount + account2.Account.GetBalance(0).Amount;
var assetStartBalance = account1.Account.GetBalance(1).Amount + account2.Account.GetBalance(1).Amount;
executor(() =>
{
foreach (var trade in testTradeResults)
{
Global.Exchange.ExecuteOrder(trade.Value);
}
});
//cleanup orders
foreach (var account in new[] { account1, account2 })
{
var activeOrders = Global.Exchange.OrderMap.GetAllAccountOrders(account);
foreach (var order in activeOrders)
{
var decodedOrderId = OrderIdConverter.Decode(order.OrderId);
new OrderRemovedEffectProccessor(new OrderRemovedEffect
{
OrderId = order.OrderId,
Amount = order.Amount,
QuoteAmount = order.QuoteAmount,
Price = order.Price,
Asset = decodedOrderId.Asset,
AccountWrapper = account
}, market.GetOrderbook(decodedOrderId.Side)).CommitEffect();
}
}
Assert.AreEqual(xlmStartBalance, account1.Account.GetBalance(0).Amount + account2.Account.GetBalance(0).Amount);
Assert.AreEqual(assetStartBalance, account1.Account.GetBalance(1).Amount + account2.Account.GetBalance(1).Amount);
Assert.AreEqual(0, account1.Account.GetBalance(0).Liabilities);
Assert.AreEqual(0, account1.Account.GetBalance(1).Liabilities);
Assert.AreEqual(0, account2.Account.GetBalance(0).Liabilities);
Assert.AreEqual(0, account2.Account.GetBalance(1).Liabilities);
}
public async Task <int> Update(DiffObject update)
{
var stellarUpdates = GetStellarDataUpdate(update.StellarInfoData);
var accountUpdates = GetAccountUpdates(update.Accounts.Values.ToList());
var balanceUpdates = GetBalanceUpdates(update.Balances.Values.ToList());
var orderUpdates = GetOrderUpdates(update.Orders.Values.ToList());
var quanta = update.Quanta.Select(a => a.Quantum);
var effects = update.Quanta.SelectMany(a => a.Effects.Values);
var retries = 1;
var maxTries = 5;
var isCommitInvoked = false;
while (true)
{
using (var session = await client.StartSessionAsync())
{
try
{
session.StartTransaction();
try
{
var updateTasks = new List <Task>();
if (stellarUpdates != null)
{
updateTasks.Add(constellationStateCollection.BulkWriteAsync(session, new WriteModel <ConstellationState>[] { stellarUpdates }));
}
if (update.ConstellationSettings != null)
{
updateTasks.Add(settingsCollection.InsertOneAsync(session, update.ConstellationSettings));
}
if (update.Assets != null && update.Assets.Count > 0)
{
updateTasks.Add(assetsCollection.InsertManyAsync(session, update.Assets));
}
if (accountUpdates != null)
{
updateTasks.Add(accountsCollection.BulkWriteAsync(session, accountUpdates));
}
if (balanceUpdates != null)
{
updateTasks.Add(balancesCollection.BulkWriteAsync(session, balanceUpdates));
}
if (orderUpdates != null)
{
updateTasks.Add(ordersCollection.BulkWriteAsync(session, orderUpdates));
}
SaveQuanta(ref updateTasks, quanta, session);
SaveEffects(ref updateTasks, effects, session);
await Task.WhenAll(updateTasks);
isCommitInvoked = true;
await CommitWithRetry(session);
break;
}
catch
{
if (!isCommitInvoked)
{
try
{
await session.AbortTransactionAsync();
}
catch { } //MongoDB best practice ;)
}
throw;
}
}
catch (Exception exc)
{
if (retries <= maxTries)
{
logger.Debug(exc, $"Error during update. {retries} try.");
retries++;
continue;
}
new Exception($"Unable to commit transaction after {retries} retries", exc);
}
}
}
return(retries);
}
public EffectProcessorsContainer(MessageEnvelope quantum, DiffObject pendingDiffObject)
{
Envelope = quantum ?? throw new ArgumentNullException(nameof(quantum));
PendingDiffObject = pendingDiffObject ?? throw new ArgumentNullException(nameof(pendingDiffObject));
QuantumModel = new QuantumItem(Apex);
}
