protected virtual int LoadStep21(XbimP21Scanner parser)
{
if (Header == null)
{
Header = new StepFileHeader(StepFileHeader.HeaderCreationMode.LeaveEmpty, this);
}
if (EntityFactory == null && _factoryResolver == null)
{
throw new XbimParserException("EntityFactory is not defined and no resolver is specified to create one. Data can't be created.");
}
parser.EntityCreate += (string name, long?label, bool header) =>
{
if (header)
{
switch (name)
{
case "FILE_DESCRIPTION":
return(Header.FileDescription);
case "FILE_NAME":
return(Header.FileName);
case "FILE_SCHEMA":
if (Header.FileSchema != null)
{
//set to new schema if it was set before from EntityFactory data
Header.FileSchema = new StepFileSchema();
}
return(Header.FileSchema);
default:
return(null);
}
}
if (EntityFactory == null)
{
EntityFactory = _factoryResolver(Header.FileSchema.Schemas);
if (EntityFactory == null)
{
throw new XbimParserException($"Entity factory resolver didn't resolve factory for schema '{string.Join(", ", Header.FileSchema.Schemas)}'");
}
InitFromEntityFactory(EntityFactory);
}
if (label == null)
{
return(EntityFactory.New(name));
}
var ent = EntityFactory.New(this, name, (int)label, true);
// if entity is null do not add so that the file load operation can survive an illegal entity
// e.g. an abstract class instantiation.
if (ent != null)
{
_instances.InternalAdd(ent);
}
else
{
var msg = $"Error in file at label {label} for type {name}.";
if (Metadata.ExpressType(name).Type.GetTypeInfo().IsAbstract)
{
msg = string.Format("Illegal element in file; cannot instantiate the abstract type {0} at label {1}.", name, label);
}
Logger?.LogError(msg);
}
//make sure that new added entities will have higher labels to avoid any clashes
if (label >= _instances.CurrentLabel)
{
_instances.CurrentLabel = (int)label;
}
return(ent);
};
try
{
parser.Parse();
//fix header with the schema if it was not a part of the data
if (Header.FileSchema.Schemas.Count == 0)
{
foreach (var s in EntityFactory.SchemasIds)
{
Header.FileSchema.Schemas.Add(s);
}
}
}
catch (Exception e)
{
var position = parser.CurrentPosition;
throw new XbimParserException(string.Format("Parser failed on line {0}, column {1}", position.EndLine, position.EndColumn), e);
}
// if the model is empty, having just a header, entity factory might still be empty
if (EntityFactory == null)
{
EntityFactory = _factoryResolver(Header.FileSchema.Schemas);
if (EntityFactory == null)
{
throw new XbimParserException($"Entity factory resolver didn't resolve factory for schema '{string.Join(", ", Header.FileSchema.Schemas)}'");
}
InitFromEntityFactory(EntityFactory);
}
//fix case if necessary
for (int i = 0; i < Header.FileSchema.Schemas.Count; i++)
{
var id = Header.FileSchema.Schemas[i];
var sid = EntityFactory.SchemasIds.FirstOrDefault(s => id.StartsWith(s, StringComparison.OrdinalIgnoreCase));
if (sid == null)
{
//add in a bit of flexibility for old Ifc models with weird schema names
var old2xSchemaNamesThatAreOK = new[] { "IFC2X2_FINAL", "IFC2X2" };
if (old2xSchemaNamesThatAreOK.FirstOrDefault(s => string.Equals(s, id, StringComparison.OrdinalIgnoreCase)) == null)
{
throw new XbimParserException("Mismatch between schema defined in the file and schemas available in the data model.");
}
else
{
sid = EntityFactory.SchemasIds.FirstOrDefault(s => string.Equals(s, "IFC2X3", StringComparison.OrdinalIgnoreCase));
}
}
//if the case is different set it to the one from entity factory
if (id != sid)
{
Header.FileSchema.Schemas[i] = sid;
}
}
return(parser.ErrorCount);
}
protected override Task HandleMessageAsync(IEnumerable <Action <UserDataEventArgs> > callbacks, string stream, string json, CancellationToken token = default)
{
if (!Users.ContainsKey(stream))
{
Logger?.LogError($"{nameof(UserDataClient)}.{nameof(HandleMessageAsync)}: Unknown listen key (\"{stream}\"). [thread: {Thread.CurrentThread.ManagedThreadId}]");
return(Task.CompletedTask); // ignore.
}
var user = Users[stream];
//Logger?.LogDebug($"{nameof(UserDataClient)}: \"{json}\"");
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
var eventTime = jObject["E"].Value <long>().ToDateTime();
// ReSharper disable once ConvertIfStatementToSwitchStatement
if (eventType == "outboundAccountInfo")
{
var commissions = new AccountCommissions(
jObject["m"].Value <int>(), // maker
jObject["t"].Value <int>(), // taker
jObject["b"].Value <int>(), // buyer
jObject["s"].Value <int>()); // seller
var status = new AccountStatus(
jObject["T"].Value <bool>(), // can trade
jObject["W"].Value <bool>(), // can withdraw
jObject["D"].Value <bool>()); // can deposit
var balances = jObject["B"]
.Select(entry => new AccountBalance(
entry["a"].Value <string>(), // asset
entry["f"].Value <decimal>(), // free amount
entry["l"].Value <decimal>())) // locked amount
.ToList();
var eventArgs = new AccountUpdateEventArgs(eventTime, token, new AccountInfo(user, commissions, status, jObject["u"].Value <long>().ToDateTime(), balances));
try
{
// ReSharper disable once InconsistentlySynchronizedField
if (_accountUpdateSubscribers.TryGetValue(stream, out var subscribers))
{
foreach (var subcriber in subscribers)
{
subcriber(eventArgs);
}
}
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
AccountUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataClient)}: Unhandled account update event handler exception.");
}
}
}
else if (eventType == "executionReport")
{
var order = new Order(user);
FillOrder(order, jObject);
var executionType = ConvertOrderExecutionType(jObject["x"].Value <string>());
var rejectedReason = ConvertOrderRejectedReason(jObject["r"].Value <string>());
var newClientOrderId = jObject["c"].Value <string>();
if (executionType == OrderExecutionType.Trade) // trade update event.
{
var trade = new AccountTrade(
jObject["s"].Value <string>(), // symbol
jObject["t"].Value <long>(), // ID
jObject["i"].Value <long>(), // order ID
jObject["L"].Value <decimal>(), // price (price of last filled trade)
jObject["z"].Value <decimal>(), // quantity (accumulated quantity of filled trades)
jObject["n"].Value <decimal>(), // commission
jObject["N"].Value <string>(), // commission asset
jObject["T"].Value <long>()
.ToDateTime(), // time
order.Side == OrderSide.Buy, // is buyer
jObject["m"].Value <bool>(), // is buyer maker
jObject["M"].Value <bool>()); // is best price
var quantityOfLastFilledTrade = jObject["l"].Value <decimal>();
var eventArgs = new AccountTradeUpdateEventArgs(eventTime, token, order, rejectedReason, newClientOrderId, trade, quantityOfLastFilledTrade);
try
{
// ReSharper disable once InconsistentlySynchronizedField
if (_accountTradeUpdateSubscribers.TryGetValue(stream, out var subscribers))
{
foreach (var subcriber in subscribers)
{
subcriber(eventArgs);
}
}
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
TradeUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataClient)}: Unhandled trade update event handler exception.");
}
}
}
else // order update event.
{
var eventArgs = new OrderUpdateEventArgs(eventTime, token, order, executionType, rejectedReason, newClientOrderId);
try
{
// ReSharper disable once InconsistentlySynchronizedField
if (_orderUpdateSubscribers.TryGetValue(stream, out var subscribers))
{
foreach (var subcriber in subscribers)
{
subcriber(eventArgs);
}
}
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
OrderUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataClient)}: Unhandled order update event handler exception.");
}
}
}
}
else
{
Logger?.LogWarning($"{nameof(UserDataClient)}.{nameof(HandleMessageAsync)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataClient)}.{nameof(HandleMessageAsync)}");
}
}
return(Task.CompletedTask);
}
protected override void OnWebSocketEvent(WebSocketStreamEventArgs args, IEnumerable <Action <TradeEventArgs> > callbacks)
{
Logger?.LogDebug($"{nameof(TradeWebSocketClient)}: \"{args.Json}\"");
try
{
var jObject = JObject.Parse(args.Json);
var eventType = jObject["e"].Value <string>();
if (eventType == "trade")
{
var eventTime = jObject["E"].Value <long>().ToDateTimeK();
var trade = new Trade(
jObject["s"].Value <string>(), // symbol
jObject["t"].Value <long>(), // trade ID
jObject["p"].Value <decimal>(), // price
jObject["q"].Value <decimal>(), // quantity
jObject["b"].Value <long>(), // buyer order ID
jObject["a"].Value <long>(), // seller order ID
jObject["T"].Value <long>()
.ToDateTimeK(), // trade time
jObject["m"].Value <bool>(), // is buyer the market maker?
jObject["M"].Value <bool>()); // is best price match?
var eventArgs = new TradeEventArgs(eventTime, args.Token, trade);
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
Trade?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(TradeWebSocketClient)}: Unhandled aggregate trade event handler exception.");
}
}
}
else
{
Logger?.LogWarning($"{nameof(TradeWebSocketClient)}.{nameof(OnWebSocketEvent)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(TradeWebSocketClient)}.{nameof(OnWebSocketEvent)}");
}
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="timestampProvider">The timestamp provider.</param>
/// <param name="rateLimiter">The rate limiter (auto configured).</param>
/// <param name="options">The options.</param>
/// <param name="logger">The logger.</param>
internal BinanceHttpClient(ITimestampProvider timestampProvider = null, IApiRateLimiter rateLimiter = null, IOptions <BinanceApiOptions> options = null, ILogger <BinanceHttpClient> logger = null)
: base(logger)
{
TimestampProvider = timestampProvider ?? new TimestampProvider();
RateLimiter = rateLimiter ?? new ApiRateLimiter();
var apiOptions = options?.Value ?? new BinanceApiOptions();
DefaultRecvWindow = apiOptions.RecvWindowDefault ?? default;
TimestampProvider.TimestampOffsetRefreshPeriod = TimeSpan.FromMinutes(apiOptions.TimestampOffsetRefreshPeriodMinutes);
try
{
// Configure request rate limiter.
RateLimiter.Configure(TimeSpan.FromMinutes(apiOptions.RequestRateLimit.DurationMinutes), apiOptions.RequestRateLimit.Count);
// Configure request burst rate limiter.
RateLimiter.Configure(TimeSpan.FromSeconds(apiOptions.RequestRateLimit.BurstDurationSeconds), apiOptions.RequestRateLimit.BurstCount);
}
catch (Exception e)
{
var message = $"{nameof(BinanceHttpClient)}: Failed to configure request rate limiter.";
Logger?.LogError(e, message);
throw new BinanceApiException(message, e);
}
var uri = new Uri(EndpointUrl);
try
{
_httpClient = new HttpClient
{
BaseAddress = uri,
Timeout = TimeSpan.FromSeconds(apiOptions.HttpClientTimeoutDefaultSeconds)
};
}
catch (Exception e)
{
var message = $"{nameof(BinanceHttpClient)}: Failed to create HttpClient.";
Logger?.LogError(e, message);
throw new BinanceApiException(message, e);
}
if (apiOptions.ServicePointManagerConnectionLeaseTimeoutMilliseconds > 0)
{
try
{
// FIX: Singleton HttpClient doesn't respect DNS changes.
// https://github.com/dotnet/corefx/issues/11224
var sp = ServicePointManager.FindServicePoint(uri);
sp.ConnectionLeaseTimeout = apiOptions.ServicePointManagerConnectionLeaseTimeoutMilliseconds;
}
catch (Exception e)
{
var message = $"{nameof(BinanceHttpClient)}: Failed to set {nameof(ServicePointManager)}.ConnectionLeaseTimeout.";
Logger?.LogError(e, message);
throw new BinanceApiException(message, e);
}
}
try
{
var version = GetType().Assembly.GetName().Version;
var versionString = $"{version.Major}.{version.Minor}.{version.Build}{(version.Revision > 0 ? $".{version.Revision}" : string.Empty)}";
_httpClient.DefaultRequestHeaders.Add("User-Agent", $"Binance/{versionString} (.NET; +https://github.com/sonvister/Binance)");
}
catch (Exception e)
{
var message = $"{nameof(BinanceHttpClient)}: Failed to set User-Agent.";
Logger?.LogError(e, message);
throw new BinanceApiException(message, e);
}
}
public override async Task StreamAsync(Uri uri, CancellationToken token)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
if (!token.CanBeCanceled)
{
throw new ArgumentException("Token must be capable of being in the canceled state.", nameof(token));
}
token.ThrowIfCancellationRequested();
if (IsStreaming)
{
throw new InvalidOperationException($"{nameof(WebSocket4NetClient)}.{nameof(StreamAsync)}: Already streaming (this method is not reentrant).");
}
IsStreaming = true;
Exception exception = null;
var tcs = new TaskCompletionSource <object>();
token.Register(() => tcs.TrySetCanceled());
var webSocket = new WebSocket4Net.WebSocket(uri.AbsoluteUri);
webSocket.Opened += (s, e) =>
{
_isOpen = true;
OnOpen();
};
webSocket.Closed += (s, e) => tcs.TrySetCanceled();
webSocket.MessageReceived += (s, evt) =>
{
try
{
var json = evt.Message;
if (!string.IsNullOrWhiteSpace(json))
{
OnMessage(json, uri.AbsolutePath);
}
else
{
Logger?.LogWarning($"{nameof(WebSocket4NetClient)}.MessageReceived: Received empty JSON message.");
}
}
//catch (OperationCanceledException) { /* ignored */ }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(WebSocket4NetClient)}.MessageReceived: WebSocket read exception.");
exception = e;
tcs.TrySetCanceled();
}
}
};
webSocket.Error += (s, e) =>
{
if (token.IsCancellationRequested)
{
return;
}
Logger?.LogError(e.Exception, $"{nameof(WebSocket4NetClient)}.Error: WebSocket exception.");
exception = e.Exception;
tcs.TrySetCanceled();
};
try
{
webSocket.Open();
await tcs.Task
.ConfigureAwait(false);
if (exception != null)
{
throw exception;
}
}
//catch (OperationCanceledException) { /* ignored */ }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(WebSocket4NetClient)}.{nameof(StreamAsync)}: WebSocket open exception.");
throw;
}
}
finally
{
if (webSocket.State == WebSocket4Net.WebSocketState.Open)
{
try { webSocket.Close(); }
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(WebSocket4NetClient)}.{nameof(StreamAsync)}: WebSocket close exception.");
}
}
webSocket.Dispose();
if (_isOpen)
{
_isOpen = false;
OnClose();
}
IsStreaming = false;
}
}
public override async Task StreamAsync(Uri uri, CancellationToken token)
{
Throw.IfNull(uri, nameof(uri));
if (!token.CanBeCanceled)
{
throw new ArgumentException($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Token must be capable of being in the canceled state.", nameof(token));
}
token.ThrowIfCancellationRequested();
if (IsStreaming)
{
throw new InvalidOperationException($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Already streaming (this method is not reentrant).");
}
IsStreaming = true;
var webSocket = new ClientWebSocket();
webSocket.Options.KeepAliveInterval = TimeSpan.FromSeconds(30);
try
{
try
{
await webSocket.ConnectAsync(uri, token)
.ConfigureAwait(false);
if (webSocket.State != WebSocketState.Open)
{
throw new Exception($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket connect failed.");
}
_isOpen = true;
RaiseOpenEvent();
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket connect exception.");
throw;
}
}
var bytes = new byte[ReceiveBufferSize];
var buffer = new ArraySegment <byte>(bytes);
var stringBuilder = new StringBuilder();
while (!token.IsCancellationRequested)
{
stringBuilder.Clear();
try
{
WebSocketReceiveResult result;
do
{
if (webSocket.State != WebSocketState.Open)
{
throw new Exception($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket is not open (state: {webSocket.State}).");
}
result = await webSocket
.ReceiveAsync(buffer, token)
.ConfigureAwait(false);
switch (result.MessageType)
{
case WebSocketMessageType.Close:
throw new Exception(result.CloseStatus.HasValue
? $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket closed ({result.CloseStatus.Value}): \"{result.CloseStatusDescription ?? "[no reason provided]"}\""
: $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket closed: \"{result.CloseStatusDescription ?? "[no reason provided]"}\"");
case WebSocketMessageType.Text when result.Count > 0:
stringBuilder.Append(Encoding.UTF8.GetString(bytes, 0, result.Count));
break;
case WebSocketMessageType.Binary:
Logger?.LogWarning($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Received unsupported binary message type.");
break;
default:
throw new ArgumentOutOfRangeException(nameof(result.MessageType), $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Unknown result message type ({result.MessageType}).");
}
}while (!result.EndOfMessage);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket receive exception.");
throw;
}
}
if (token.IsCancellationRequested)
{
continue;
}
var json = stringBuilder.ToString();
if (!string.IsNullOrWhiteSpace(json))
{
RaiseMessageEvent(new WebSocketClientEventArgs(json));
}
else
{
Logger?.LogWarning($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Received empty JSON message.");
}
}
}
finally
{
// NOTE: WebSocketState.CloseSent should not be encountered since CloseOutputAsync is not used.
if (webSocket.State == WebSocketState.Open || webSocket.State == WebSocketState.CloseReceived)
{
try
{
await webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, string.Empty, CancellationToken.None)
.ConfigureAwait(false);
}
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket close exception.");
}
}
webSocket?.Dispose();
if (_isOpen)
{
_isOpen = false;
RaiseCloseEvent();
}
IsStreaming = false;
Logger?.LogDebug($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Task complete. [thread: {Thread.CurrentThread.ManagedThreadId}]");
}
}
protected override async Task ProcessJsonAsync(string json, CancellationToken token = default)
{
try
{
string streamName = null;
//IJsonStreamObserver[] subscribers;
// NOTE: Avoid locking... allowing for eventual consistency of subscribers.
//lock (_sync)
//{
if (json.IsJsonObject())
{
var jObject = JObject.Parse(json);
// Get stream name.
streamName = jObject["stream"]?.Value <string>();
if (streamName != null)
{
// Get JSON data.
var data = jObject["data"]?.ToString(Formatting.None);
if (data == null)
{
Logger?.LogError($"{nameof(BinanceWebSocketStream)}: No JSON 'data' in message. [thread: {Thread.CurrentThread.ManagedThreadId}]");
return; // ignore.
}
json = data;
}
}
if (streamName == null)
{
// Get stream name.
streamName = StreamNames.FirstOrDefault();
if (streamName == null)
{
Logger?.LogError($"{nameof(BinanceWebSocketStream)}: No subscribed streams. [thread: {Thread.CurrentThread.ManagedThreadId}]");
return; // ignore.
}
}
if (!Subscribers.TryGetValue(streamName, out var observers))
{
return; // ignore.
}
// Get subscribers.
var subscribers = observers?.ToArray();
//}
await NotifyListenersAsync(subscribers, streamName, json, token)
.ConfigureAwait(false);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(BinanceWebSocketStream)}: Failed processing JSON message. [thread: {Thread.CurrentThread.ManagedThreadId}]");
}
}
}
/// <summary>
/// Creates a perpetual lock on a message by continuously renewing it's lock.
/// This is usually created at the start of a handler so that we guarantee that we still have a valid lock
/// and we retain that lock until we finish handling the message.
/// </summary>
/// <param name="message">The message that we want to create the lock on.</param>
/// <param name="messageBody">The message body.</param>
/// <returns>Task.</returns>
internal async Task <bool> Lock(Message message, T messageBody)
{
try
{
await Receiver.RenewLockAsync(message.SystemProperties.LockToken).ConfigureAwait(false);
// Remove before adding if already exists.
LockTimers.TryRemove(messageBody, out _);
LockTimers.TryAdd(
messageBody,
new Timer(
async _ =>
{
if (!Receiver.IsClosedOrClosing && Messages.TryGetValue(messageBody, out var msg))
{
try
{
await Receiver.RenewLockAsync(msg.SystemProperties.LockToken).ConfigureAwait(false);
}
catch (ObjectDisposedException ex)
{
Logger?.LogWarning(ex, "Error during message lock, service bus client disposed");
Release(messageBody);
}
catch (Exception ex) when(ex is MessageLockLostException || ex is MessageNotFoundException)
{
Logger?.LogWarning(ex, "Error during message lock, lock was lost [THIS CAN BE IGNORED - already in use or already processed]");
Release(messageBody);
}
catch (Exception e)
{
Logger?.LogError(e, "Error during message lock");
Release(messageBody);
}
}
else
{
Release(messageBody);
}
},
null,
TimeSpan.FromSeconds(Config.ReceiverInfo.LockRenewalTimeThreshold),
TimeSpan.FromSeconds(Config.ReceiverInfo.LockRenewalTimeThreshold)));
return(true);
}
catch (Exception ex) when(ex is MessageLockLostException || ex is MessageNotFoundException)
{
BrokenReceiverCheck(); // Fallback check as we shouldnt have this error here.
Logger?.LogWarning(ex, "Error during message lock, fallback iniatied");
Release(messageBody);
return(false);
}
catch (Exception e)
{
Logger?.LogError(e, "Error during message lock");
// Catch all other types (timeout, http etc).
MessageIn?.OnError(e);
Release(messageBody);
return(false);
}
}
protected override int Process(IEnumerable <IBaseEntity> datas, dynamic sender = null)
{
if (datas == null || datas.Count() == 0)
{
return(0);
}
var tableInfo = new TableInfo(datas.First().GetType());
IDbConnection conn = null;
for (int i = 0; i < RetryTimes; ++i)
{
try
{
conn = RefreshConnectionString();
// 每天执行一次建表操作, 可以实现每天一个表的操作，或者按周分表可以在运行时创建新表。
var key = tableInfo.Schema.TableNamePostfix != TableNamePostfix.None ? $"{tableInfo.Schema.TableName}_{DateTime.Now:yyyyMMdd}" : tableInfo.Schema.TableName;
Sqls sqls;
lock (this)
{
if (_sqls.ContainsKey(key))
{
sqls = _sqls[key];
}
else
{
sqls = GenerateSqls(tableInfo);
_sqls.Add(key, sqls);
InitDatabaseAndTable(conn, tableInfo);
}
}
int count = 0;
switch (_pipelineMode)
{
case PipelineMode.Insert:
{
count += conn.MyExecute(sqls.InsertSql, datas);
break;
}
case PipelineMode.InsertAndIgnoreDuplicate:
{
count += conn.MyExecute(sqls.InsertAndIgnoreDuplicateSql, datas);
break;
}
case PipelineMode.InsertNewAndUpdateOld:
{
count += conn.MyExecute(sqls.InsertNewAndUpdateOldSql, datas);
break;
}
case PipelineMode.Update:
{
if (string.IsNullOrWhiteSpace(sqls.UpdateSql))
{
Logger?.LogError("Check your TableInfo attribute contains UpdateColumns value.");
throw new SpiderException("UpdateSql is null.");
}
count += conn.MyExecute(sqls.UpdateSql, datas);
break;
}
default:
{
count += conn.MyExecute(sqls.InsertSql, datas);
break;
}
}
return(count);
}
catch (Exception e)
{
if (RetryExceptionMessages.Any(m => e.Message.Contains(m)))
{
Thread.Sleep(5000);
continue;
}
throw;
}
finally
{
conn?.Dispose();
}
}
throw new SpiderException($"Pipeline process failed.");
}
/// <summary>
/// Deserialize JSON and raise <see cref="SymbolStatisticsTradeEventArgs"/> event.
/// </summary>
/// <param name="json"></param>
/// <param name="token"></param>
/// <param name="callback"></param>
protected override void DeserializeJsonAndRaiseEvent(string json, CancellationToken token, Action <SymbolStatisticsEventArgs> callback = null)
{
Throw.IfNullOrWhiteSpace(json, nameof(json));
Logger?.LogDebug($"{nameof(SymbolStatisticsWebSocketClient)}: \"{json}\"");
try
{
SymbolStatisticsEventArgs eventArgs;
if (json.IsJsonArray())
{
// Simulate a single event time.
var eventTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var statistics = JArray.Parse(json).Select(jToken => DeserializeSymbolStatistics(jToken)).ToArray();
eventArgs = new SymbolStatisticsEventArgs(eventTime, token, statistics);
}
else
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
if (eventType == "24hrTicker")
{
var eventTime = jObject["E"].Value <long>();
var statistics = DeserializeSymbolStatistics(jObject);
eventArgs = new SymbolStatisticsEventArgs(eventTime, token, statistics);
}
else
{
Logger?.LogWarning($"{nameof(SymbolStatisticsWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}: Unexpected event type ({eventType}).");
return;
}
}
try
{
callback?.Invoke(eventArgs);
StatisticsUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsWebSocketClient)}: Unhandled aggregate trade event handler exception.");
}
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}");
}
}
}
public bool Parse(bool onlyHeader = false)
{
var skipping = SkipTypes.Any();
var eofToken = (int)Tokens.EOF;
var tok = _scanner.yylex();
int endEntityToken = ';';
while (tok != eofToken && !Cancel)
{
try
{
if (tok >= 63)
{
Tokens t = (Tokens)tok;
switch (t)
{
case Tokens.HEADER:
BeginHeader();
break;
case Tokens.ENDSEC:
if (_inHeader && onlyHeader)
{
return(true);
}
EndSec();
break;
case Tokens.DATA:
BeginData();
break;
case Tokens.ENTITY:
NewEntity(_scanner.yylval.strVal);
break;
case Tokens.TYPE:
var type = _scanner.yylval.strVal;
if (skipping && SkipTypes.Contains(type))
{
var current = _processStack.Pop();
//SkipEntities.Add(current.EntityLabel);
while (tok != endEntityToken)
{
tok = _scanner.yylex();
}
break;
}
if (!SetType(type))
{
// move to the end of entity if we couldn't create it
while (tok != endEntityToken)
{
tok = _scanner.yylex();
}
}
break;
case Tokens.INTEGER:
SetIntegerValue(_scanner.yylval.strVal);
break;
case Tokens.FLOAT:
SetFloatValue(_scanner.yylval.strVal);
break;
case Tokens.STRING:
SetStringValue(_scanner.yylval.strVal);
break;
case Tokens.BOOLEAN:
SetBooleanValue(_scanner.yylval.strVal);
break;
case Tokens.IDENTITY:
SetObjectValue(_scanner.yylval.strVal);
break;
case Tokens.HEXA:
SetHexValue(_scanner.yylval.strVal);
break;
case Tokens.ENUM:
SetEnumValue(_scanner.yylval.strVal);
break;
case Tokens.NONDEF:
SetNonDefinedValue();
break;
case Tokens.OVERRIDE:
SetOverrideValue();
break;
case Tokens.TEXT:
case Tokens.error:
case Tokens.ILLEGALCHAR:
throw new XbimParserException($"Unexpected scanner token {t.ToString()}, line {_scanner.yylloc.StartLine}, column {_scanner.yylloc.StartColumn}");
case Tokens.SCOPE:
case Tokens.ENDSCOPE:
case Tokens.ISOSTEPSTART:
case Tokens.ISOSTEPEND:
case Tokens.MISC:
case Tokens.EOF:
default:
break;
}
}
else
{
char c = (char)tok;
switch (c)
{
case '(':
BeginList();
break;
case ')':
EndList();
break;
case ';':
EndEntity();
break;
case '/':
case ',':
case '=':
default:
break;
}
}
// get next token
tok = _scanner.yylex();
}
//XbimParserException is a reason to terminate execution
catch (XbimParserException e)
{
Logger?.LogError(LogEventIds.ParserFailure, e, e.Message);
return(false);
}
//other exceptions might occure but those should just make the parser to wait for the next start of entity
//and start from there
catch (Exception e)
{
Logger?.LogError(LogEventIds.FailedEntity, e, e.Message);
ErrorCount++;
// clear current entity stack to make sure there are no residuals
_processStack.Clear();
// scan until the beginning of next entity
var entityToken = (int)Tokens.ENTITY;
while (tok != eofToken && tok != entityToken)
{
tok = _scanner.yylex();
}
}
}
EndParse();
return(ErrorCount == 0);
}
public IEntryPoint Generate(string operationName, string?executionTarget,
RuntimeCapability?runtimeCapability = null)
{
Logger?.LogDebug($"Generating entry point: operationName={operationName}, executionTarget={executionTarget}");
var logger = new QSharpLogger(Logger);
var compilerMetadata = References.CompilerMetadata;
// Clear references to previously-built assemblies
WorkspaceAssemblies = Array.Empty <AssemblyInfo>();
SnippetsAssemblyInfo = null;
EntryPointAssemblyInfo = null;
// Compile the workspace against the provided execution target
var workspaceFiles = Workspace.SourceFiles.ToArray();
if (workspaceFiles.Any())
{
Logger?.LogDebug($"{workspaceFiles.Length} files found in workspace. Compiling.");
var workspaceAssemblies = new List <AssemblyInfo>();
foreach (var project in Workspace.Projects.Where(p => p.SourceFiles.Any()))
{
try
{
workspaceAssemblies.Add(Compiler.BuildFiles(
project.SourceFiles.ToArray(),
compilerMetadata.WithAssemblies(workspaceAssemblies.ToArray()),
logger,
Path.Combine(Workspace.CacheFolder, $"__entrypoint{project.CacheDllName}")));
}
catch (Exception e)
{
logger.LogError(
"IQS004",
$"Error compiling project {project.ProjectFile} for execution target {executionTarget}: {e.Message}");
}
}
if (!workspaceAssemblies.Any() || logger.HasErrors)
{
Logger?.LogError($"Error compiling workspace.");
throw new CompilationErrorsException(logger);
}
WorkspaceAssemblies = workspaceAssemblies.ToArray();
compilerMetadata = compilerMetadata.WithAssemblies(WorkspaceAssemblies);
}
// Compile the snippets against the provided execution target
var snippets = Snippets.Items.ToArray();
if (snippets.Any())
{
Logger?.LogDebug($"{snippets.Length} items found in snippets. Compiling.");
SnippetsAssemblyInfo = Compiler.BuildSnippets(
snippets, compilerMetadata, logger, Path.Combine(Workspace.CacheFolder, "__entrypoint__snippets__.dll"));
if (SnippetsAssemblyInfo == null || logger.HasErrors)
{
Logger?.LogError($"Error compiling snippets.");
throw new CompilationErrorsException(logger);
}
compilerMetadata = compilerMetadata.WithAssemblies(SnippetsAssemblyInfo);
}
// Build the entry point assembly
var operationInfo = new EntryPointOperationResolver(this).Resolve(operationName);
if (operationInfo == null)
{
Logger?.LogError($"{operationName} is not a recognized Q# operation name.");
throw new UnsupportedOperationException(operationName);
}
EntryPointAssemblyInfo = Compiler.BuildEntryPoint(
operationInfo, compilerMetadata, logger, Path.Combine(Workspace.CacheFolder, "__entrypoint__.dll"), executionTarget, runtimeCapability);
if (EntryPointAssemblyInfo == null || logger.HasErrors)
{
Logger?.LogError($"Error compiling entry point for operation {operationName}.");
throw new CompilationErrorsException(logger);
}
if (EntryPointAssemblyInfo.Operations.Count() <= 1)
{
// Entry point assembly contained zero or one operations; this
// may indicate that C# code is not being correctly
// regenerated. At least two operations (the entry point and
// the operation called from the entry point) are expected.
Logger?.LogWarning(
"Internal error compiling entry point for operation {OperationName}; entry point assembly did not contain the right number of operations. This should never happen, and most likely indicates a bug in IQ#. ",
operationName
);
}
var entryPointOperations = EntryPointAssemblyInfo
.Operations
.Where(op => op.Header.Attributes.Any(
attr =>
{
var qName = attr.TypeId.ValueOr(null);
return(qName != null &&
qName.Name == "EntryPoint" &&
qName.Namespace == "Microsoft.Quantum.Core");
}
));
var entryPointOperationInfo = entryPointOperations
.SingleOrDefault();
if (entryPointOperationInfo == null)
{
throw new Exception($"Entry point assembly contained {entryPointOperations.Count()}, but expected 1.");
}
// Construct the EntryPointInfo<,> object
var parameterTypes = entryPointOperationInfo.RoslynParameters.Select(p => p.ParameterType).ToArray();
var typeCount = parameterTypes.Length;
Type entryPointInputType = typeCount switch
{
0 => typeof(QVoid),
1 => parameterTypes.Single(),
_ => PartialMapper.TupleTypes[typeCount].MakeGenericType(parameterTypes)
};
Type entryPointOutputType = entryPointOperationInfo.ReturnType;
Type entryPointInfoType = typeof(EntryPointInfo <,>).MakeGenericType(new Type[] { entryPointInputType, entryPointOutputType });
var entryPointInfo = entryPointInfoType.GetConstructor(new Type[] { typeof(Type) })
.Invoke(new object[] { entryPointOperationInfo.RoslynType });
return(new EntryPoint(entryPointInfo, entryPointInputType, entryPointOutputType, entryPointOperationInfo));
}
}
protected override void OnWebSocketEvent(WebSocketStreamEventArgs args, IEnumerable <Action <DepthUpdateEventArgs> > callbacks)
{
Logger?.LogDebug($"{nameof(DepthWebSocketClient)}: \"{args.Json}\"");
try
{
var jObject = JObject.Parse(args.Json);
var eventType = jObject["e"]?.Value <string>();
DepthUpdateEventArgs eventArgs;
switch (eventType)
{
case null: // partial depth stream.
{
var symbol = args.StreamName.Split('@')[0].ToUpperInvariant();
// Simulate event time.
var eventTime = DateTime.UtcNow.ToTimestampK().ToDateTimeK();
var lastUpdateId = jObject["lastUpdateId"].Value <long>();
var bids = jObject["bids"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
var asks = jObject["asks"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
eventArgs = new DepthUpdateEventArgs(eventTime, args.Token, symbol, lastUpdateId, lastUpdateId, bids, asks);
break;
}
case "depthUpdate":
{
var symbol = jObject["s"].Value <string>();
var eventTime = jObject["E"].Value <long>().ToDateTimeK();
var firstUpdateId = jObject["U"].Value <long>();
var lastUpdateId = jObject["u"].Value <long>();
var bids = jObject["b"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
var asks = jObject["a"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
eventArgs = new DepthUpdateEventArgs(eventTime, args.Token, symbol, firstUpdateId, lastUpdateId, bids, asks);
break;
}
default:
Logger?.LogWarning($"{nameof(DepthWebSocketClient)}.{nameof(OnWebSocketEvent)}: Unexpected event type ({eventType}).");
return;
}
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
DepthUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DepthWebSocketClient)}: Unhandled depth update event handler exception.");
}
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DepthWebSocketClient)}.{nameof(OnWebSocketEvent)}");
}
}
}
public IEntryPoint Generate(string operationName, string?executionTarget,
RuntimeCapability?runtimeCapability = null)
{
Logger?.LogDebug($"Generating entry point: operationName={operationName}, executionTarget={executionTarget}");
var logger = new QSharpLogger(Logger);
var compilerMetadata = References.CompilerMetadata;
// Clear references to previously-built assemblies
WorkspaceAssemblies = Array.Empty <AssemblyInfo>();
SnippetsAssemblyInfo = null;
EntryPointAssemblyInfo = null;
// Compile the workspace against the provided execution target
var workspaceFiles = Workspace.SourceFiles.ToArray();
if (workspaceFiles.Any())
{
Logger?.LogDebug($"{workspaceFiles.Length} files found in workspace. Compiling.");
var workspaceAssemblies = new List <AssemblyInfo>();
foreach (var project in Workspace.Projects.Where(p => p.SourceFiles.Any()))
{
try
{
workspaceAssemblies.Add(Compiler.BuildFiles(
project.SourceFiles.ToArray(),
compilerMetadata.WithAssemblies(workspaceAssemblies.ToArray()),
logger,
Path.Combine(Workspace.CacheFolder, $"__entrypoint{project.CacheDllName}"),
executionTarget,
runtimeCapability));
}
catch (Exception e)
{
logger.LogError(
"IQS004",
$"Error compiling project {project.ProjectFile} for execution target {executionTarget}: {e.Message}");
}
}
if (!workspaceAssemblies.Any() || logger.HasErrors)
{
Logger?.LogError($"Error compiling workspace.");
throw new CompilationErrorsException(logger.Errors.ToArray());
}
WorkspaceAssemblies = workspaceAssemblies.ToArray();
compilerMetadata = compilerMetadata.WithAssemblies(WorkspaceAssemblies);
}
// Compile the snippets against the provided execution target
var snippets = Snippets.Items.ToArray();
if (snippets.Any())
{
Logger?.LogDebug($"{snippets.Length} items found in snippets. Compiling.");
SnippetsAssemblyInfo = Compiler.BuildSnippets(
snippets, compilerMetadata, logger, Path.Combine(Workspace.CacheFolder, "__entrypoint__snippets__.dll"), executionTarget, runtimeCapability);
if (SnippetsAssemblyInfo == null || logger.HasErrors)
{
Logger?.LogError($"Error compiling snippets.");
throw new CompilationErrorsException(logger.Errors.ToArray());
}
compilerMetadata = compilerMetadata.WithAssemblies(SnippetsAssemblyInfo);
}
// Build the entry point assembly
var operationInfo = new EntryPointOperationResolver(this).Resolve(operationName);
if (operationInfo == null)
{
Logger?.LogError($"{operationName} is not a recognized Q# operation name.");
throw new UnsupportedOperationException(operationName);
}
EntryPointAssemblyInfo = Compiler.BuildEntryPoint(
operationInfo, compilerMetadata, logger, Path.Combine(Workspace.CacheFolder, "__entrypoint__.dll"), executionTarget, runtimeCapability);
if (EntryPointAssemblyInfo == null || logger.HasErrors)
{
Logger?.LogError($"Error compiling entry point for operation {operationName}.");
throw new CompilationErrorsException(logger.Errors.ToArray());
}
var entryPointOperationInfo = EntryPointAssemblyInfo.Operations.Single();
// Construct the EntryPointInfo<,> object
var parameterTypes = entryPointOperationInfo.RoslynParameters.Select(p => p.ParameterType).ToArray();
var typeCount = parameterTypes.Length;
Type entryPointInputType = typeCount switch
{
0 => typeof(QVoid),
1 => parameterTypes.Single(),
_ => PartialMapper.TupleTypes[typeCount].MakeGenericType(parameterTypes)
};
Type entryPointOutputType = entryPointOperationInfo.ReturnType;
Type entryPointInfoType = typeof(EntryPointInfo <,>).MakeGenericType(new Type[] { entryPointInputType, entryPointOutputType });
var entryPointInfo = entryPointInfoType.GetConstructor(new Type[] { typeof(Type) })
.Invoke(new object[] { entryPointOperationInfo.RoslynType });
return(new EntryPoint(entryPointInfo, entryPointInputType, entryPointOutputType, entryPointOperationInfo));
}
}
/// <summary>
/// Deserialize JSON and raise <see cref="CandlestickEventArgs"/> event.
/// </summary>
/// <param name="json"></param>
/// <param name="symbol"></param>
/// <param name="token"></param>
/// <param name="callback"></param>
protected override void DeserializeJsonAndRaiseEvent(string json, Symbol symbol, CancellationToken token,
Action <CandlestickEventArgs> callback = null)
{
Throw.IfNullOrWhiteSpace(json, nameof(json));
Logger?.LogDebug($"{nameof(CandlestickWebSocketClient)}: \"{json}\"");
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
if (eventType == "kline")
{
//var symbol = jObject["s"].Value<string>();
var eventTime = jObject["E"].Value <long>();
var firstTradeId = jObject["k"]["f"].Value <long>();
var lastTradeId = jObject["k"]["L"].Value <long>();
var isFinal = jObject["k"]["x"].Value <bool>();
var candlestick = new Candlestick(
jObject["k"]["s"].Value <string>(), // symbol
jObject["k"]["i"].Value <string>()
.ToCandlestickInterval(), // interval
jObject["k"]["t"].Value <long>(), // open time
jObject["k"]["o"].Value <decimal>(), // open
jObject["k"]["h"].Value <decimal>(), // high
jObject["k"]["l"].Value <decimal>(), // low
jObject["k"]["c"].Value <decimal>(), // close
jObject["k"]["v"].Value <decimal>(), // volume
jObject["k"]["T"].Value <long>(), // close time
jObject["k"]["q"].Value <decimal>(), // quote asset volume
jObject["k"]["n"].Value <long>(), // number of trades
jObject["k"]["V"].Value <decimal>(), // taker buy base asset volume (volume of active buy)
jObject["k"]["Q"].Value <decimal>() // taker buy quote asset volume (quote volume of active buy)
);
var eventArgs = new CandlestickEventArgs(eventTime, token, candlestick, firstTradeId, lastTradeId, isFinal);
try
{
callback?.Invoke(eventArgs);
Candlestick?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(CandlestickWebSocketClient)}: Unhandled candlestick event handler exception.");
}
}
}
else
{
Logger?.LogWarning($"{nameof(CandlestickWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(CandlestickWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}");
}
}
}
[ExcludeFromCodeCoverage] // Skipped - too many edge cases that cant be tested.
internal void Read([MaybeNull] object _, int readBatchSize)
{
lock (_objLock)
{
// If already processing, return from read
// (shouldn't happen, but in for a safeguard).
if (_batchInProcess)
{
return;
}
if (Receiver == null)
{
throw new InvalidOperationException("Receiver Entity has not been configured");
}
// Set batch "in processing" by setting flag and stopping read timer.
_batchInProcess = true;
StopReading();
try
{
var messages = Receiver.ReceiveAsync(readBatchSize).GetAwaiter().GetResult();
if (messages == null)
{
// If we have null returned, double check we aren't getting null when there are
// actually items on the entity - BUG FIX FOR SB.
BrokenReceiverCheck();
return; // no more processing.
}
foreach (var message in messages)
{
// Get the message content from the body.
var messageBody = GetTypedMessageContent(message);
if (messageBody != null)
{
Messages[messageBody] = message;
// auto lock when message is received and set renewal timer.
Lock(message, messageBody).GetAwaiter().GetResult();
}
}
/***********************/
// NOTE: Re-add the task.run code to increase parallelism... for now its one-by-one processing.
// Raise the event for processing.
foreach (var message in Messages)
{
// After message locked, raise process message event.
MessageIn?.OnNext(message.Key);
}
}
catch (ObjectDisposedException dex)
{
Logger?.LogError(dex, "Trying to read when receiver is disposed - IGNORE");
}
catch (InvalidOperationException iex)
{
Logger?.LogError(iex, "Error during read of service bus message");
MessageIn?.OnError(iex);
}
catch (Exception ex) when(ex is MessageLockLostException || ex is MessageNotFoundException || ex is ObjectDisposedException)
{
Logger?.LogWarning(ex, "Error during message lock, lock was lost [THIS CAN BE IGNORED - already in use or already processed]");
}
catch (Exception e)
{
Logger?.LogError(e, "Error during read of service bus message");
// Error on all other types of exception (http, timeout etc).
MessageIn?.OnError(e);
}
finally
{
if (!Disposed)
{
// Set batch processing to finished and restart the read timer.
_batchInProcess = false;
StartReading(readBatchSize);
}
}
}
}
protected override void OnWebSocketEvent(WebSocketStreamEventArgs args, IEnumerable <Action <SymbolStatisticsEventArgs> > callbacks)
{
Logger?.LogDebug($"{nameof(SymbolStatisticsWebSocketClient)}: \"{args.Json}\"");
try
{
SymbolStatisticsEventArgs eventArgs;
if (args.Json.IsJsonArray())
{
// Simulate a single event time.
var eventTime = DateTime.UtcNow.ToTimestamp().ToDateTime();
var statistics = JArray.Parse(args.Json).Select(DeserializeSymbolStatistics).ToArray();
eventArgs = new SymbolStatisticsEventArgs(eventTime, args.Token, statistics);
}
else
{
var jObject = JObject.Parse(args.Json);
var eventType = jObject["e"].Value <string>();
if (eventType == "24hrTicker")
{
var eventTime = jObject["E"].Value <long>().ToDateTime();
var statistics = DeserializeSymbolStatistics(jObject);
eventArgs = new SymbolStatisticsEventArgs(eventTime, args.Token, statistics);
}
else
{
Logger?.LogWarning($"{nameof(SymbolStatisticsWebSocketClient)}.{nameof(OnWebSocketEvent)}: Unexpected event type ({eventType}).");
return;
}
}
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
StatisticsUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsWebSocketClient)}: Unhandled aggregate trade event handler exception.");
}
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsWebSocketClient)}.{nameof(OnWebSocketEvent)}");
}
}
}
/// <summary>
/// Deserialize JSON and raise <see cref="DepthUpdateEventArgs"/> event.
/// </summary>
/// <param name="json"></param>
/// <param name="symbol"></param>
/// <param name="token"></param>
/// <param name="callback"></param>
protected override void DeserializeJsonAndRaiseEvent(string json, Symbol symbol, CancellationToken token,
Action <DepthUpdateEventArgs> callback = null)
{
Throw.IfNullOrWhiteSpace(json, nameof(json));
Logger?.LogDebug($"{nameof(DepthWebSocketClient)}: \"{json}\"");
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"]?.Value <string>();
DepthUpdateEventArgs eventArgs;
switch (eventType)
{
case null:
{
// Simulate event time.
var eventTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var lastUpdateId = jObject["lastUpdateId"].Value <long>();
var bids = jObject["bids"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
var asks = jObject["asks"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
eventArgs = new DepthUpdateEventArgs(eventTime, token, symbol, lastUpdateId, lastUpdateId, bids, asks);
break;
}
case "depthUpdate":
{
var symbolString = jObject["s"].Value <string>();
var eventTime = jObject["E"].Value <long>();
var firstUpdateId = jObject["U"].Value <long>();
var lastUpdateId = jObject["u"].Value <long>();
var bids = jObject["b"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
var asks = jObject["a"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
eventArgs = new DepthUpdateEventArgs(eventTime, token, symbolString, firstUpdateId, lastUpdateId, bids, asks);
break;
}
default:
Logger?.LogWarning($"{nameof(DepthWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}: Unexpected event type ({eventType}).");
return;
}
try
{
callback?.Invoke(eventArgs);
DepthUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DepthWebSocketClient)}: Unhandled depth update event handler exception.");
}
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DepthWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}");
}
}
}
public void ModifyField(string fieldName, string value)
{
Logger?.LogTrace("ModifyField Entity: " +
GetType().FullName + "." +
MethodBase.GetCurrentMethod()?.Name +
" - " + fieldName + " - " + value);
AddEntityModifyFieldEvent(fieldName, value);
try
{
var field = GetType().GetProperty(fieldName); //, BindingFlags.Public | BindingFlags.Instance);
if (field != null && field.CanWrite)
{
//Console.WriteLine("1 " + field.Name);
var t = Nullable.GetUnderlyingType(field.PropertyType) ?? field.PropertyType;
object safeValue = null;
//If is byte[]
if (t == typeof(byte[]))
{
if (value != null)
{
var splitResult = value.Split('-');
if (splitResult.Length > 0)
{
var byteArray = new byte[splitResult.Length];
for (var i = 0; i < splitResult.Length; i++)
{
byteArray[i] = Convert.ToByte(splitResult[i], 16);
}
safeValue = byteArray;
}
else
{
Logger?.LogWarning("ModifyField Entity: " +
GetType().FullName + "." +
MethodBase.GetCurrentMethod()?.Name +
" - " + fieldName + " - " + value + ": Byte Array on data!");
}
}
}
//if (t == typeof(Guid))
//{
// ////T t = (T)TypeDescriptor.GetConverter(typeof(T)).ConvertFromInvariantString(text);
// //safeValue = (value == null)
// // ? null
// // : TypeDescriptor.GetConverter(t).ConvertFromInvariantString(value);
//}
else
{
//Console.WriteLine("2");
//safeValue = (value == null) ? null : Convert.ChangeType(value, t);
safeValue = (value == null)
? null
: TypeDescriptor.GetConverter(t).ConvertFromInvariantString(value);
}
field.SetValue(this, safeValue, null);
//Console.WriteLine("3");
}
else
{
Logger?.LogWarning("ModifyField Entity: " +
GetType().FullName + "." +
MethodBase.GetCurrentMethod()?.Name +
" - " + fieldName + " - " + value + ": Can not write OR null!");
}
}
catch (Exception ex)
{
Logger?.LogError(ex.Source + " " + ex.Message + " " + ex.StackTrace);
}
}
protected override async Task <Entry> LoadEntryAsync(Id entryId, IProgress <double> progress, CancellationToken cancellationToken)
{
progress?.Report(0);
if (Id.IsNullOrDefault(entryId))
{
throw new ArgumentException("The given id was the default id.", nameof(entryId));
}
Entry entry = DefaultEntry.Instance;
switch (entryId.Type)
{
case IdType.Movie:
TmdbMovie tmdbMovie;
try
{
tmdbMovie = await _client.GetMovieAsync(entryId.Number, (MovieMethods)4239, cancellationToken);
}
catch (Exception e)
{
Logger?.LogError($"An error occured while retrieving {nameof(Movie)} corresponding to {entryId}.", e);
break;
}
entry = _tmdbConverter.ConvertMovie(tmdbMovie);
break;
case IdType.Artist:
Person tmdbPerson;
try
{
tmdbPerson = await _client.GetPersonAsync(entryId.Number, (PersonMethods)31, cancellationToken);
}
catch (Exception e)
{
Logger?.LogError($"An error occured while retrieving {nameof(Artist)} corresponding to {entryId}.", e);
break;
}
entry = _tmdbConverter.ConvertPerson(tmdbPerson);
break;
case IdType.TvSeries:
TvShow tmdbSeries;
try
{
tmdbSeries = await _client.GetTvShowAsync(entryId.Number, (TvShowMethods)127, cancellationToken : cancellationToken);
}
catch (Exception e)
{
Logger?.LogError($"An error occured while retrieving artist corresponding to {entryId}.", e);
break;
}
entry = _tmdbConverter.ConvertTvSeries(tmdbSeries);
break;
default:
Logger?.LogError($"{nameof(IdType)} {entryId.Type.ToString()} is not handled for {entryId}.");
break;
}
progress?.Report(1);
return(entry);
}
/// <summary>
///
/// </summary>
/// <param name="event"></param>
/// <returns></returns>
protected override async Task <OrderBookCacheEventArgs> OnAction(DepthUpdateEventArgs @event)
{
if (_limit > 0)
{
// Ignore events with same or earlier order book update.
if (_orderBookClone != null && @event.LastUpdateId <= _orderBookClone.LastUpdateId)
{
return(null);
}
// Top <level> bids and asks, pushed every second.
// NOTE: LastUpdateId is not contiguous between events when using partial depth stream.
_orderBookClone = new OrderBook(_symbol, @event.LastUpdateId, @event.Bids, @event.Asks);
}
else
{
// If order book has not been initialized.
if (_orderBook == null)
{
// Synchronize.
await SynchronizeOrderBookAsync()
.ConfigureAwait(false);
}
// Ignore events prior to order book snapshot.
if (@event.LastUpdateId <= _orderBook.LastUpdateId)
{
return(null);
}
// If there is a gap in events (order book out-of-sync).
// ReSharper disable once InvertIf
if (@event.FirstUpdateId > _orderBook.LastUpdateId + 1)
{
Logger?.LogError($"{nameof(OrderBookCache)}: Synchronization failure (first update ID > last update ID + 1).");
await Task.Delay(1000, Token)
.ConfigureAwait(false); // wait a bit.
// Re-synchronize.
await SynchronizeOrderBookAsync()
.ConfigureAwait(false);
// If still out-of-sync.
// ReSharper disable once InvertIf
if (@event.FirstUpdateId > _orderBook.LastUpdateId + 1)
{
Logger?.LogError($"{nameof(OrderBookCache)}: Re-Synchronization failure (first update ID > last update ID + 1).");
// Reset and wait for next event.
_orderBook = null;
return(null);
}
}
Logger?.LogDebug($"{nameof(OrderBookCache)}: Updating order book [last update ID: {@event.LastUpdateId}].");
_orderBook.Modify(@event.LastUpdateId, @event.Bids, @event.Asks);
_orderBookClone = _orderBook.Clone();
}
return(new OrderBookCacheEventArgs(_orderBookClone));
}
private async static Task Main(string[] args)
{
try
{
if (!ParseArguments(args))
{
return;
}
logger = new Logger(options?.LogFilePath, finishCTS.Token)
{
ShowDebugLogs = options?.ShowDebugLogs ?? false
};
if (checkUpdates || !options.DisableUpdates)
{
if (UpdateChecker.IsStartingUpdate(logger, checkUpdates) || checkUpdates)
{
return;
}
}
logger.LogTechState("Loading data from file...");
try
{
LoadFile(options.FileName);
}
catch (Exception ex)
{
logger.LogError($"Error during loading file: {ex.Message}");
}
logger.LogInfo($"File is loaded: {w}x{h}, {deltas.Count + 1} frames");
logger.LogTechState("Downloading palette...");
PixelPlanetHttpApi api = new PixelPlanetHttpApi
{
ProxySettings = proxySettings
};
UserModel user = await api.GetMeAsync();
logger.LogTechInfo("Palette retrieved");
CanvasModel canvas = user.Canvases[canvasType];
palette = new Palette(canvas.Colors);
DirectoryInfo dir = Directory.CreateDirectory($"seq_{startTime:MM.dd_HH-mm}");
string pathTemplate = Path.Combine(dir.FullName, "{0:MM.dd_HH-mm-ss}.png");
try
{
SaveLoadedData(pathTemplate);
}
catch (Exception ex)
{
logger.LogError($"Error during saving results: {ex.Message}");
}
}
catch (Exception ex)
{
logger?.LogError($"Unhandled app level exception: {ex.Message}");
logger?.LogDebug(ex.ToString());
}
finally
{
if (logger != null)
{
logger.LogInfo("Exiting...");
logger.LogInfo($"Logs were saved to {logger.LogFilePath}");
}
finishCTS.Cancel();
if (logger != null)
{
Thread.Sleep(500);
}
logger?.Dispose();
finishCTS.Dispose();
Console.ForegroundColor = ConsoleColor.White;
Environment.Exit(0);
}
}
public override async Task StreamAsync(Uri uri, CancellationToken token)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
if (!token.CanBeCanceled)
{
throw new ArgumentException("Token must be capable of being in the canceled state.", nameof(token));
}
if (token.IsCancellationRequested)
{
return;
}
lock (_sync)
{
if (IsStreaming)
{
throw new InvalidOperationException($"{nameof(WebSocketSharpClient)}.{nameof(StreamAsync)}: Already streaming (this method is not reentrant).");
}
IsStreaming = true;
}
Exception exception = null;
var tcs = new TaskCompletionSource <object>();
using (token.Register(() => tcs.TrySetCanceled()))
{
var webSocket = new WebSocketSharp.WebSocket(uri.AbsoluteUri);
webSocket.SslConfiguration.EnabledSslProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12;
webSocket.Log.Output = (d, s) => { /* ignore */ };
webSocket.OnOpen += (s, e) =>
{
OnOpen();
};
webSocket.OnClose += (s, e) => tcs.TrySetCanceled();
webSocket.OnMessage += (s, evt) =>
{
try
{
if (!evt.IsText)
{
return;
}
var json = evt.Data;
if (!string.IsNullOrWhiteSpace(json))
{
OnMessage(json, uri.AbsolutePath);
}
else
{
Logger?.LogWarning($"{nameof(WebSocketSharpClient)}.OnMessage: Received empty JSON message.");
}
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(WebSocketSharpClient)}.OnMessage: WebSocket read exception.");
exception = e;
tcs.TrySetCanceled();
}
}
};
webSocket.OnError += (s, e) =>
{
if (token.IsCancellationRequested)
{
return;
}
Logger?.LogError(e.Exception, $"{nameof(WebSocketSharpClient)}.OnError: WebSocket exception.");
exception = e.Exception;
tcs.TrySetCanceled();
};
try
{
webSocket.Connect();
await tcs.Task
.ConfigureAwait(false);
if (exception != null)
{
throw exception;
}
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(WebSocketSharpClient)}.{nameof(StreamAsync)}: WebSocket connect exception.");
throw;
}
}
finally
{
if (webSocket.IsAlive)
{
try { webSocket.Close(CloseStatusCode.Normal); }
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(WebSocketSharpClient)}.{nameof(StreamAsync)}: WebSocket close exception.");
}
}
((IDisposable)webSocket).Dispose();
if (IsOpen)
{
OnClose();
}
IsStreaming = false;
}
}
}
private async Task <string> RequestAsync(HttpMethod method, BinanceHttpRequest request, CancellationToken token = default)
{
Throw.IfNull(request, nameof(request));
token.ThrowIfCancellationRequested();
var requestMessage = request.CreateMessage(method);
Logger?.LogDebug($"{nameof(BinanceHttpClient)}.{nameof(RequestAsync)}: [{method.Method}] \"{requestMessage.RequestUri}\"");
using (var response = await _httpClient.SendAsync(requestMessage, token).ConfigureAwait(false))
{
if (response.IsSuccessStatusCode)
{
var json = await response.Content.ReadAsStringAsync()
.ConfigureAwait(false);
OnMessage(json, requestMessage.RequestUri.AbsoluteUri);
return(json);
}
if (response.StatusCode == HttpStatusCode.GatewayTimeout)
{
throw new BinanceUnknownStatusException();
}
var error = await response.Content.ReadAsStringAsync()
.ConfigureAwait(false);
var errorCode = 0;
string errorMessage = null;
// ReSharper disable once InvertIf
if (!string.IsNullOrWhiteSpace(error) && error.IsJsonObject())
{
try // to parse server error response.
{
var jObject = JObject.Parse(error);
errorCode = jObject["code"]?.Value <int>() ?? 0;
errorMessage = jObject["msg"]?.Value <string>();
}
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(BinanceHttpClient)}.{nameof(RequestAsync)} failed to parse server error response: \"{error}\"");
}
}
OnMessage(error, requestMessage.RequestUri.AbsoluteUri);
// ReSharper disable once SwitchStatementMissingSomeCases
switch (response.StatusCode)
{
case (HttpStatusCode)429:
throw new BinanceRequestRateLimitExceededException(response.ReasonPhrase, errorCode, errorMessage);
case (HttpStatusCode)418:
throw new BinanceRequestRateLimitIpBanException(response.ReasonPhrase, errorCode, errorMessage);
default:
throw new BinanceHttpException(response.StatusCode, response.ReasonPhrase, errorCode, errorMessage);
}
}
}
protected override void HandleMessage(IEnumerable <Action <DepthUpdateEventArgs> > callbacks, string stream, string json)
{
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"]?.Value <string>();
DepthUpdateEventArgs eventArgs;
switch (eventType)
{
case null: // partial depth stream.
{
var symbol = stream.Split('@')[0].ToUpperInvariant();
// Simulate event time.
var eventTime = DateTime.UtcNow.ToTimestamp().ToDateTime();
var lastUpdateId = jObject["lastUpdateId"].Value <long>();
var bids = jObject["bids"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
var asks = jObject["asks"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
eventArgs = new DepthUpdateEventArgs(eventTime, symbol, lastUpdateId, lastUpdateId, bids, asks);
break;
}
case "depthUpdate":
{
var symbol = jObject["s"].Value <string>();
var eventTime = jObject["E"].Value <long>().ToDateTime();
var firstUpdateId = jObject["U"].Value <long>();
var lastUpdateId = jObject["u"].Value <long>();
var bids = jObject["b"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
var asks = jObject["a"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToArray();
eventArgs = new DepthUpdateEventArgs(eventTime, symbol, firstUpdateId, lastUpdateId, bids, asks);
break;
}
default:
Logger?.LogWarning($"{nameof(DepthClient)}.{nameof(HandleMessage)}: Unexpected event type ({eventType}).");
return;
}
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
DepthUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogWarning(e, $"{nameof(DepthClient)}.{nameof(HandleMessage)}: Unhandled depth update event handler exception.");
}
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(DepthClient)}.{nameof(HandleMessage)}");
}
}
protected override async Task <Response> ImplDownloadAsync(Request request)
{
var response = new Response
{
Request = request
};
for (int i = 0; i < RetryTime; ++i)
{
HttpResponseMessage httpResponseMessage = null;
WebProxy proxy = null;
try
{
var httpRequestMessage = GenerateHttpRequestMessage(request);
if (UseProxy)
{
if (HttpProxyPool == null)
{
response.Exception = "HttpProxyPool is null";
response.Success = false;
Logger?.LogError(
$"{request.OwnerId} download {request.Url} failed [{i}]: {response.Exception}");
return(response);
}
else
{
proxy = HttpProxyPool.GetProxy();
if (proxy == null)
{
response.Exception = "There is no available proxy";
response.Success = false;
Logger?.LogError(
$"{request.OwnerId} download {request.Url} failed [{i}]: {response.Exception}");
return(response);
}
}
}
var httpClientEntry = GetHttpClientEntry(proxy == null ? "DEFAULT" : $"{proxy.Address}", proxy);
httpResponseMessage = Framework.NetworkCenter == null
? await httpClientEntry.HttpClient.SendAsync(httpRequestMessage)
: await Framework.NetworkCenter.Execute(async() =>
await httpClientEntry.HttpClient.SendAsync(httpRequestMessage));
httpResponseMessage.EnsureSuccessStatusCode();
response.TargetUrl = httpResponseMessage.RequestMessage.RequestUri.AbsoluteUri;
var bytes = httpResponseMessage.Content.ReadAsByteArrayAsync().Result;
if (!ExcludeMediaTypes.Any(t =>
httpResponseMessage.Content.Headers.ContentType.MediaType.Contains(t)))
{
if (!DownloadFile)
{
StorageFile(request, bytes);
}
}
else
{
var content = ReadContent(request, bytes,
httpResponseMessage.Content.Headers.ContentType.CharSet);
if (DecodeHtml)
{
#if NETFRAMEWORK
content = System.Web.HttpUtility.UrlDecode(
System.Web.HttpUtility.HtmlDecode(content),
string.IsNullOrEmpty(request.Encoding)
? Encoding.UTF8
: Encoding.GetEncoding(request.Encoding));
#else
content = WebUtility.UrlDecode(WebUtility.HtmlDecode(content));
#endif
}
response.RawText = content;
}
if (!string.IsNullOrWhiteSpace(request.ChangeIpPattern) &&
Regex.IsMatch(response.RawText, request.ChangeIpPattern))
{
if (UseProxy)
{
response.TargetUrl = null;
response.RawText = null;
response.Success = false;
// 把代理设置为空，影响 final 代码块里不作归还操作，等于删除此代理
proxy = null;
}
else
{
// 不支持切换 IP
if (Framework.NetworkCenter == null ||
!Framework.NetworkCenter.SupportAdsl)
{
response.Success = false;
response.Exception = "IP Banded";
Logger?.LogError(
$"{request.OwnerId} download {request.Url} failed [{i}]: {response.Exception}");
return(response);
}
else
{
Framework.NetworkCenter.Redial();
}
}
}
else
{
response.Success = true;
Logger?.LogInformation(
$"{request.OwnerId} download {request.Url} success");
return(response);
}
}
catch (Exception e)
{
response.Exception = e.Message;
response.Success = false;
Logger?.LogError($"{request.OwnerId} download {request.Url} failed [{i}]: {e}");
}
finally
{
if (HttpProxyPool != null && proxy != null)
{
HttpProxyPool.ReturnProxy(proxy,
httpResponseMessage?.StatusCode ?? HttpStatusCode.ServiceUnavailable);
}
try
{
httpResponseMessage?.Dispose();
}
catch (Exception e)
{
Logger?.LogWarning($"{request.OwnerId} dispose response {request.Url} failed [{i}]: {e}");
}
}
// 下载失败需要等待一秒，防止频率过高。
// TODO: 改成可配置
Thread.Sleep(1000);
}
return(response);
}
protected override async Task StoreAsync(DataContext context, Dictionary <Type, List <dynamic> > dict)
{
using var conn = TryCreateDbConnection();
foreach (var kv in dict)
{
var list = (IList)kv.Value;
var tableMetadata = _tableMetadataDict.GetOrAdd(kv.Key,
type => ((IEntity)list[0]).GetTableMetadata());
var sqlStatements = GetSqlStatements(tableMetadata);
// 因为同一个存储器会收到不同的数据对象，因此不能使用 initAsync 来初始化数据库
_executedCache.GetOrAdd(sqlStatements.CreateTableSql, str =>
{
var conn1 = TryCreateDbConnection();
EnsureDatabaseAndTableCreated(conn1, sqlStatements);
return(string.Empty);
});
for (var i = 0; i < RetryTimes; ++i)
{
IDbTransaction transaction = null;
try
{
if (UseTransaction)
{
transaction = conn.BeginTransaction();
}
switch (Mode)
{
case StorageMode.Insert:
{
await conn.ExecuteAsync(sqlStatements.InsertSql, list, transaction);
break;
}
case StorageMode.InsertIgnoreDuplicate:
{
await conn.ExecuteAsync(sqlStatements.InsertIgnoreDuplicateSql, list, transaction);
break;
}
case StorageMode.Update:
{
if (string.IsNullOrWhiteSpace(sqlStatements.UpdateSql))
{
throw new SpiderException("未能生成更新 SQL");
}
await conn.ExecuteAsync(sqlStatements.UpdateSql, list, transaction);
break;
}
case StorageMode.InsertAndUpdate:
{
await conn.ExecuteAsync(sqlStatements.InsertAndUpdateSql, list, transaction);
break;
}
}
transaction?.Commit();
break;
}
catch (Exception ex)
{
Logger.LogError($"尝试插入数据失败: {ex}");
// 网络异常需要重试，并且不需要 Rollback
var endOfStreamException = ex.InnerException as EndOfStreamException;
if (endOfStreamException == null)
{
try
{
transaction?.Rollback();
}
catch (Exception e)
{
Logger?.LogError($"数据库回滚失败: {e}");
}
break;
}
}
finally
{
transaction?.Dispose();
}
}
}
}
protected override void OnWebSocketEvent(WebSocketStreamEventArgs args, IEnumerable <Action <UserDataEventArgs> > callbacks)
{
var user = _listenKeys.First(_ => _.Value == args.StreamName).Key;
Logger?.LogDebug($"{nameof(UserDataWebSocketClient)}: \"{args.Json}\"");
try
{
var jObject = JObject.Parse(args.Json);
var eventType = jObject["e"].Value <string>();
var eventTime = jObject["E"].Value <long>().ToDateTime();
// ReSharper disable once ConvertIfStatementToSwitchStatement
if (eventType == "outboundAccountInfo")
{
var commissions = new AccountCommissions(
jObject["m"].Value <int>(), // maker
jObject["t"].Value <int>(), // taker
jObject["b"].Value <int>(), // buyer
jObject["s"].Value <int>()); // seller
var status = new AccountStatus(
jObject["T"].Value <bool>(), // can trade
jObject["W"].Value <bool>(), // can withdraw
jObject["D"].Value <bool>()); // can deposit
var balances = jObject["B"]
.Select(entry => new AccountBalance(
entry["a"].Value <string>(), // asset
entry["f"].Value <decimal>(), // free amount
entry["l"].Value <decimal>())) // locked amount
.ToList();
var eventArgs = new AccountUpdateEventArgs(eventTime, args.Token, new AccountInfo(user, commissions, status, jObject["u"].Value <long>().ToDateTime(), balances));
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
AccountUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}: Unhandled account update event handler exception.");
}
}
}
else if (eventType == "executionReport")
{
var order = new Order(user);
FillOrder(order, jObject);
var executionType = ConvertOrderExecutionType(jObject["x"].Value <string>());
var rejectedReason = ConvertOrderRejectedReason(jObject["r"].Value <string>());
var newClientOrderId = jObject["c"].Value <string>();
if (executionType == OrderExecutionType.Trade) // trade update event.
{
var trade = new AccountTrade(
jObject["s"].Value <string>(), // symbol
jObject["t"].Value <long>(), // ID
jObject["i"].Value <long>(), // order ID
jObject["L"].Value <decimal>(), // price (price of last filled trade)
jObject["z"].Value <decimal>(), // quantity (accumulated quantity of filled trades)
jObject["n"].Value <decimal>(), // commission
jObject["N"].Value <string>(), // commission asset
jObject["T"].Value <long>()
.ToDateTime(), // time
order.Side == OrderSide.Buy, // is buyer
jObject["m"].Value <bool>(), // is buyer maker
jObject["M"].Value <bool>()); // is best price
var quantityOfLastFilledTrade = jObject["l"].Value <decimal>();
var eventArgs = new AccountTradeUpdateEventArgs(eventTime, args.Token, order, rejectedReason, newClientOrderId, trade, quantityOfLastFilledTrade);
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
TradeUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}: Unhandled trade update event handler exception.");
}
}
}
else // order update event.
{
var eventArgs = new OrderUpdateEventArgs(eventTime, args.Token, order, executionType, rejectedReason, newClientOrderId);
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
OrderUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}: Unhandled order update event handler exception.");
}
}
}
}
else
{
Logger?.LogWarning($"{nameof(UserDataWebSocketClient)}.{nameof(OnWebSocketEvent)}: Unexpected event type ({eventType}) - \"{args.Json}\"");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}.{nameof(OnWebSocketEvent)}");
}
}
}
protected override async Task <DataFlowResult> Store(DataFlowContext context)
{
IDbConnection conn = TryCreateDbConnection(context);
using (conn)
{
foreach (var item in context.GetParseItems())
{
var tableMetadata = (TableMetadata)context[item.Key];
SqlStatements sqlStatements = GetSqlStatements(tableMetadata);
lock (this)
{
EnsureDatabaseAndTableCreated(conn, sqlStatements);
}
for (int i = 0; i < RetryTimes; ++i)
{
IDbTransaction transaction = null;
try
{
if (UseTransaction)
{
transaction = conn.BeginTransaction();
}
var list = item.Value;
switch (StorageType)
{
case StorageType.Insert:
{
await conn.ExecuteAsync(sqlStatements.InsertSql, list, transaction);
break;
}
case StorageType.InsertIgnoreDuplicate:
{
await conn.ExecuteAsync(sqlStatements.InsertIgnoreDuplicateSql, list, transaction);
break;
}
case StorageType.Update:
{
if (string.IsNullOrWhiteSpace(sqlStatements.UpdateSql))
{
throw new SpiderException("未能生成更新 SQL");
}
await conn.ExecuteAsync(sqlStatements.UpdateSql, list, transaction);
break;
}
case StorageType.InsertAndUpdate:
{
await conn.ExecuteAsync(sqlStatements.InsertAndUpdateSql, list, transaction);
break;
}
}
transaction?.Commit();
break;
}
catch (Exception ex)
{
Logger?.LogError($"尝试插入数据失败: {ex}");
// 网络异常需要重试，并且不需要 Rollback
var endOfStreamException = ex.InnerException as EndOfStreamException;
if (endOfStreamException == null)
{
try
{
transaction?.Rollback();
}
catch (Exception e)
{
Logger?.LogError($"数据库回滚失败: {e}");
}
break;
}
}
finally
{
transaction?.Dispose();
}
}
}
}
return(DataFlowResult.Success);
}
protected override void HandleMessage(IEnumerable <Action <CandlestickEventArgs> > callbacks, string stream, string json)
{
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
if (eventType == "kline")
{
//var symbol = jObject["s"].Value<string>();
var eventTime = jObject["E"].Value <long>().ToDateTime();
var kLine = jObject["k"];
var firstTradeId = kLine["f"].Value <long>();
var lastTradeId = kLine["L"].Value <long>();
var isFinal = kLine["x"].Value <bool>();
var candlestick = new Candlestick(
kLine["s"].Value <string>(), // symbol
kLine["i"].Value <string>() // interval
.ToCandlestickInterval(),
kLine["t"].Value <long>() // open time
.ToDateTime(),
kLine["o"].Value <decimal>(), // open
kLine["h"].Value <decimal>(), // high
kLine["l"].Value <decimal>(), // low
kLine["c"].Value <decimal>(), // close
kLine["v"].Value <decimal>(), // volume
kLine["T"].Value <long>() // close time
.ToDateTime(),
kLine["q"].Value <decimal>(), // quote asset volume
kLine["n"].Value <long>(), // number of trades
kLine["V"].Value <decimal>(), // taker buy base asset volume (volume of active buy)
kLine["Q"].Value <decimal>() // taker buy quote asset volume (quote volume of active buy)
);
var eventArgs = new CandlestickEventArgs(eventTime, candlestick, firstTradeId, lastTradeId, isFinal);
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
Candlestick?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogWarning(e, $"{nameof(CandlestickClient)}.{nameof(HandleMessage)}: Unhandled candlestick event handler exception.");
}
}
else
{
Logger?.LogWarning($"{nameof(CandlestickClient)}.{nameof(HandleMessage)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(CandlestickClient)}.{nameof(HandleMessage)}");
}
}
public void LogError(Exception ex)
{
Logger logger = new Logger();
logger.Storage = Logger.StorageType.File;
logger.LogError(ex, _dsn, Page, false);
}
