private string GenerateSelectSql(TableInfo tableInfo)
{
if (tableInfo.Primary.Count == 0)
{
Logger?.LogWarning("Can't generate select sql, primary key is missing.");
return(null);
}
var tableName = GetTableName(tableInfo);
var database = GetDatabaseName(tableInfo);
string where = "";
foreach (var column in tableInfo.Primary)
{
where += $" [{GetColumnName(column)}] = @{column.Name} AND";
}
where = where.Substring(0, where.Length - 3);
var sql = string.IsNullOrWhiteSpace(database) ? $"SELECT * FROM [{tableName}] WHERE {where};" :
$"USE [{database}]; SELECT * FROM [{tableName}] WHERE {where};";
return(sql);
}
/// <summary>
/// 生成更新数据的 SQL 语句
/// </summary>
/// <param name="tableMetadata">表元数据</param>
/// <returns>SQL 语句</returns>
protected virtual string GenerateUpdateSql(TableMetadata tableMetadata)
{
if (tableMetadata.Updates == null || tableMetadata.Updates.Count == 0)
{
Logger?.LogWarning("实体没有设置主键, 无法生成 Update 语句");
return(null);
}
var where = "";
foreach (var column in tableMetadata.Primary)
{
where += $" {Escape}{GetNameSql(column)}{Escape} = @{column} AND";
}
where = where.Substring(0, where.Length - 3);
var setCols = string.Join(", ",
tableMetadata.Updates.Select(c => $"{Escape}{GetNameSql(c)}{Escape}= @{c}"));
var tableSql = GenerateTableSql(tableMetadata);
var sql = $"UPDATE {tableSql} SET {setCols} WHERE {where};";
return(sql);
}
/// <summary>
/// Completes a message by doing the actual READ from the queue.
/// </summary>
/// <param name="messages">The message we want to complete.</param>
/// <returns>Task.</returns>
internal async Task Complete(IEnumerable <T> messages)
{
try
{
var messageLockTokens = new List <string>();
foreach (var message in messages)
{
if (Messages.TryGetValue(message, out var msg))
{
Release(message);
if (msg.SystemProperties.LockedUntilUtc < DateTime.Now)
{
Receiver.RenewLockAsync(msg.SystemProperties.LockToken).GetAwaiter().GetResult();
}
}
if (msg?.SystemProperties.LockToken != null)
{
messageLockTokens.Add(msg.SystemProperties.LockToken);
}
}
if (messageLockTokens.Count > 0)
{
await Receiver.CompleteAsync(messageLockTokens).ConfigureAwait(false);
}
}
catch (Exception ex) when(ex is MessageLockLostException || ex is MessageNotFoundException)
{
Logger?.LogWarning(ex, "Error during message Complete, lock was lost [THIS CAN BE IGNORED - already in use or already processed]");
}
catch (Exception e)
{
Logger?.LogError(e, "Error during message Complete");
MessageIn?.OnError(e);
}
}
[ExcludeFromCodeCoverage] // Skipped - too many edge cases that cant be tested.
internal async Task SendMessage(T messageBody, KeyValuePair <string, object>[] messageProperties = null)
{
if (Disposed)
{
return;
}
try
{
// Converts the message to a broker message and verifies its size is within the allowed size.
var message = ConvertMessage(messageBody, messageProperties);
await Sender.SendAsync(message).ConfigureAwait(false);
}
catch (MessagingEntityDisabledException mex)
{
Logger?.LogWarning(mex, "Error occurred sending message - entity is disabled");
throw new EntityDisabledException(Config.SenderInfo.EntityName, mex.Message, mex);
}
catch (QuotaExceededException qex)
{
long maxSize = 0, currentSize = 0;
// Build a meaningful entity full message.
try
{
long.TryParse(qex.Message.Substring("Size of entity in bytes:", ",").Trim(), out currentSize);
long.TryParse(qex.Message.Substring("Max entity size in bytes:", ".").Trim(), out maxSize);
}
catch (Exception)
{
// Do nothing on error if there are any problems grabbing the size in bytes.
}
Logger?.LogWarning(qex, $"Error occurred sending messages - sender entity is full (max allowed: {maxSize}, current: {currentSize})");
throw new EntityFullException(Config.SenderInfo.EntityName, qex.Message, currentSize, maxSize, qex);
}
}
public override void HandleBasicDeliver(string consumerTag, ulong deliveryTag, bool redelivered, string exchange, string routingKey, RC.IBasicProperties properties, byte[] body)
{
Logger?.LogDebug("Storing delivery for consumer tag: {tag} with deliveryTag: {deliveryTag} for consumer: {consumer}", ConsumerTag, deliveryTag, ToString());
try
{
var delivery = new Delivery(consumerTag, new Envelope(deliveryTag, redelivered, exchange, routingKey), properties, body, QueueName);
if (Consumer.AbortStarted > 0)
{
if (!Consumer.Queue.TryAdd(delivery, Consumer.ShutdownTimeout))
{
RabbitUtils.SetPhysicalCloseRequired(Model, true);
_ = Consumer.Queue.TakeWhile((d) => Consumer.Queue.Count > 0);
if (!Canceled)
{
RabbitUtils.Cancel(Model, consumerTag);
}
try
{
Model.Close();
}
catch (Exception)
{
// Noop
}
}
}
else
{
Consumer.Queue.TryAdd(delivery);
}
}
catch (Exception e)
{
Logger?.LogWarning(e, "Unexpected exception during delivery");
}
}
internal static void ValidateListBox(ListBox element)
{
// Skip validation if a debugger isn't attached
if (!Debugger.IsAttached)
{
return;
}
if (element is ListBox lb)
{
if (lb.ItemsSource == null)
{
Logger?.LogWarning($"Cannot animate ListBox items because {nameof(lb.ItemsSource)} is null.");
return;
}
var itemSettings = GetItems(lb);
// ItemsProperty can only be set on a AnimatedListBox or AnimatedListView.
if (itemSettings != null && !(lb is AnimatedListBox) && !(lb is AnimatedListView))
{
throw new ArgumentException($"{nameof(ItemsProperty)} can only be set on a {nameof(AnimatedListBox)} or {nameof(AnimatedListView)}.");
}
// ListBox item animations will only work if the inner ScrollViewer has CanContentScroll = true
if (lb?.FindDescendant <ScrollViewer>() is ScrollViewer scroller && !scroller.CanContentScroll)
{
throw new ArgumentException($"{nameof(ListBox)} item animations will only work if the inner {nameof(ScrollViewer)} has {nameof(scroller.CanContentScroll)} = true");
}
// Don't set a value for the Event property, is it disregarded for ListBox item animations.
if (itemSettings != null && itemSettings.Event != DefaultSettings.Event)
{
throw new ArgumentException($"Don't set a value for the {nameof(itemSettings.Event)} property, is it disregarded for {nameof(ListBox)} item animations.");
}
}
}
public async Task <IImageSourceServiceResult <WImageSource>?> GetImageSourceAsync(IUriImageSource imageSource, float scale = 1, CancellationToken cancellationToken = default)
{
if (imageSource.IsEmpty)
{
return(null);
}
// TODO: use a real caching library with the URI
if (imageSource is not IStreamImageSource streamImageSource)
{
throw new InvalidOperationException("Unable to load URI as a stream.");
}
try
{
using var stream = await streamImageSource.GetStreamAsync(cancellationToken);
if (stream == null)
{
throw new InvalidOperationException("Unable to load image stream.");
}
var image = new BitmapImage();
using var ras = stream.AsRandomAccessStream();
await image.SetSourceAsync(ras);
var result = new ImageSourceServiceResult(image);
return(result);
}
catch (Exception ex)
{
Logger?.LogWarning(ex, "Unable to load image URI '{Uri}'.", imageSource.Uri);
throw;
}
}
public override async Task <IImageSourceServiceResult?> LoadDrawableAsync(IImageSource imageSource, Android.Widget.ImageView imageView, CancellationToken cancellationToken = default)
{
var streamImageSource = (IStreamImageSource)imageSource;
if (!streamImageSource.IsEmpty)
{
Stream?stream = null;
try
{
stream = await streamImageSource.GetStreamAsync(cancellationToken).ConfigureAwait(false);
var callback = new ImageLoaderCallback();
PlatformInterop.LoadImageFromStream(imageView, stream, callback);
var result = await callback.Result;
stream?.Dispose();
return(result);
}
catch (Exception ex)
{
Logger?.LogWarning(ex, "Unable to load image stream.");
throw;
}
finally
{
if (stream != null)
{
GC.KeepAlive(stream);
}
}
}
return(null);
}
public async Task <IImageSourceServiceResult <Drawable>?> GetDrawableAsync(IStreamImageSource imageSource, Context context, CancellationToken cancellationToken = default)
{
if (imageSource.IsEmpty)
{
return(null);
}
try
{
var stream = await imageSource.GetStreamAsync(cancellationToken).ConfigureAwait(false);
// We can use the .NET stream directly because we register the InputStreamModelLoader.
// There are 2 alternatives:
// - Load the bitmap manually and pass that along, but then we do not get the decoding features.
// - Copy the stream into a byte array and that is double memory usage - especially for large streams.
var inputStream = new InputStreamAdapter(stream);
var result = await Glide
.With(context)
.Load(inputStream)
.SetDiskCacheStrategy(DiskCacheStrategy.None)
.SubmitAsync(context, cancellationToken)
.ConfigureAwait(false);
if (result == null)
{
throw new InvalidOperationException("Unable to load image stream.");
}
return(result);
}
catch (Exception ex)
{
Logger?.LogWarning(ex, "Unable to load image stream.");
throw;
}
}
protected override void EndNestedType(string value)
{
try
{
PropertyValue.Init(_processStack.Pop().Entity);
CurrentInstance = _processStack.Peek();
if (CurrentInstance.Entity != null)
{
CurrentInstance.Entity.Parse(CurrentInstance.CurrentParamIndex, PropertyValue, NestedIndex);
}
}
catch (Exception)
{
if (ErrorCount > MaxErrorCount)
{
throw new XbimParserException("Too many errors in file, parser execution terminated");
}
ErrorCount++;
var mainEntity = _processStack.Last();
if (mainEntity != null)
{
Logger?.LogWarning("Entity #{0,-5} {1}, error at parameter {2} value = {4}",
mainEntity.EntityLabel, mainEntity.Entity.GetType().Name.ToUpper(),
mainEntity.CurrentParamIndex + 1,
value);
}
else
{
Logger?.LogWarning("Unhandled Parser error, in Parser.cs EndNestedType");
}
}
if (ListNestLevel == 0)
{
CurrentInstance.CurrentParamIndex++;
_deferListItems = false;
}
}
private void HandleDeclarationException(int passiveDeclareRetries, DeclarationException e)
{
if (passiveDeclareRetries > 0 && Channel.IsOpen)
{
Logger?.LogWarning(e, "Queue declaration failed; retries left={retries}", passiveDeclareRetries);
try
{
Thread.Sleep(FailedDeclarationRetryInterval);
}
catch (Exception e1)
{
Declaring = false;
ActiveObjectCounter.Release(this);
throw RabbitExceptionTranslator.ConvertRabbitAccessException(e1);
}
}
else if (e.FailedQueues.Count < Queues.Count)
{
Logger?.LogWarning("Not all queues are available; only listening on those that are - configured: {queues}; not available: {notavail}", string.Join(',', Queues), string.Join(',', e.FailedQueues));
lock (MissingQueues)
{
foreach (var q in e.FailedQueues)
{
MissingQueues.Add(q);
}
}
LastRetryDeclaration = DateTimeOffset.Now.ToUnixTimeMilliseconds();
}
else
{
Declaring = false;
ActiveObjectCounter.Release(this);
throw new QueuesNotAvailableException("Cannot prepare queue for listener. Either the queue doesn't exist or the broker will not allow us to use it.", e);
}
}
protected override void Dispose(bool disposing)
{
Logger?.LogDebug("[FtpDataStream] Disposing");
base.Dispose(disposing);
try
{
encapsulatedStream.Dispose();
if (client.Configuration.DisconnectTimeoutMilliseconds.HasValue)
{
client.ControlStream.SetTimeouts(client.Configuration.DisconnectTimeoutMilliseconds.Value);
}
client.CloseFileDataStreamAsync().Wait();
}
catch (Exception e)
{
Logger?.LogWarning(0, e, "Closing the data stream took longer than expected");
}
finally
{
client.ControlStream.ResetTimeouts();
}
}
public async Task <int> ExecuteAsync(string standardQuery, object paramValues, CommandType?dbCommandType = null,
int commandTimeout = 0)
{
try
{
Logger?.LogInformation($"Executing DB query: \n {standardQuery} ");
Logger?.LogInformation($"DB query parameters: \n {paramValues?.JsonSerialize()}");
var stopwatch = new Stopwatch();
stopwatch.Start();
var result =
await
(await GetOpenConnectionAsync()).ExecuteAsync(standardQuery, paramValues, _currentTransaction,
commandType: dbCommandType, commandTimeout: commandTimeout);
Logger?.LogInformation($"DB query results: \n {result}");
if (stopwatch.ElapsedMilliseconds >= ExecutionPerformanceThresholdInMs)
{
Logger?.LogWarning(
$"DB query execution time: \n {stopwatch.ElapsedMilliseconds} ms \n {standardQuery}",
dbCommandType,
standardQuery, paramValues);
}
else
{
Logger?.LogInformation(
$"DB query execution time: \n {stopwatch.ElapsedMilliseconds} ms \n {standardQuery}");
}
return(result);
}
catch (Exception ex)
{
Logger?.LogError($"Executing DB query failed: \n {ex.Message}", ex, standardQuery, paramValues);
throw ex;
}
}
/// <summary>
/// 生成插入新数据或者更新旧数据的 SQL 语句
/// </summary>
/// <param name="tableMetadata">表元数据</param>
/// <returns>SQL 语句</returns>
protected virtual string GenerateInsertAndUpdateSql(TableMetadata tableMetadata)
{
if (!tableMetadata.HasPrimary)
{
Logger?.LogWarning("实体没有设置主键, 无法生成 InsertAndUpdate 语句");
return(null);
}
// UPDATE MyTable SET FieldA=@FieldA WHERE Key=@Key IF @@ROWCOUNT = 0 INSERT INTO MyTable (FieldA) VALUES (@FieldA)
var columns = tableMetadata.Columns;
var isAutoIncrementPrimary = tableMetadata.IsAutoIncrementPrimary;
// 去掉自增主键
var insertColumns =
(isAutoIncrementPrimary ? columns.Where(c1 => c1.Key != tableMetadata.Primary.First()) : columns)
.ToArray();
var columnsSql = string.Join(", ", insertColumns.Select(c => $"[{GetNameSql(c.Key)}]"));
var columnsParamsSql = string.Join(", ", insertColumns.Select(p => $"@{p.Key}"));
var where = "";
foreach (var column in tableMetadata.Primary)
{
where += $" [{GetNameSql(column)}] = @{column} AND";
}
where = where.Substring(0, where.Length - 3);
var tableSql = GenerateTableSql(tableMetadata);
var setCols = string.Join(", ", insertColumns.Select(c => $"[{GetNameSql(c.Key)}] = @{c.Key}"));
var sql =
$"UPDATE {tableSql} SET {setCols} WHERE {where} IF @@ROWCOUNT = 0 INSERT INTO {tableSql} ({columnsSql}) VALUES ({columnsParamsSql});";
return(sql);
}
public async Task <IImageSourceServiceResult <Image>?> GetImageAsync(IFileImageSource imageSource, Image image, CancellationToken cancellationToken = default)
{
if (imageSource.IsEmpty)
{
return(null);
}
var filename = imageSource.File;
try
{
if (!string.IsNullOrEmpty(filename))
{
var isLoadComplated = await image.LoadAsync(GetPath(filename), cancellationToken);
if (!isLoadComplated)
{
//If it fails, call the Load function to remove the previous image.
image.Load(string.Empty);
throw new InvalidOperationException("Unable to load image file.");
}
var result = new ImageSourceServiceResult(image);
return(result);
}
else
{
throw new InvalidOperationException("Unable to load image file.");
}
}
catch (Exception ex)
{
Logger?.LogWarning(ex, "Unable to load image file '{File}'.", filename);
throw;
}
}
protected override void HandleMessage(IEnumerable <Action <CandlestickEventArgs> > callbacks, string stream, string json)
{
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
if (eventType == "kline")
{
//var symbol = jObject["s"].Value<string>();
var eventTime = jObject["E"].Value <long>().ToDateTime();
var kLine = jObject["k"];
var firstTradeId = kLine["f"].Value <long>();
var lastTradeId = kLine["L"].Value <long>();
var isFinal = kLine["x"].Value <bool>();
var candlestick = new Candlestick(
kLine["s"].Value <string>(), // symbol
kLine["i"].Value <string>() // interval
.ToCandlestickInterval(),
kLine["t"].Value <long>() // open time
.ToDateTime(),
kLine["o"].Value <decimal>(), // open
kLine["h"].Value <decimal>(), // high
kLine["l"].Value <decimal>(), // low
kLine["c"].Value <decimal>(), // close
kLine["v"].Value <decimal>(), // volume
kLine["T"].Value <long>() // close time
.ToDateTime(),
kLine["q"].Value <decimal>(), // quote asset volume
kLine["n"].Value <long>(), // number of trades
kLine["V"].Value <decimal>(), // taker buy base asset volume (volume of active buy)
kLine["Q"].Value <decimal>() // taker buy quote asset volume (quote volume of active buy)
);
var eventArgs = new CandlestickEventArgs(eventTime, candlestick, firstTradeId, lastTradeId, isFinal);
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
Candlestick?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogWarning(e, $"{nameof(CandlestickClient)}.{nameof(HandleMessage)}: Unhandled candlestick event handler exception.");
}
}
else
{
Logger?.LogWarning($"{nameof(CandlestickClient)}.{nameof(HandleMessage)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(CandlestickClient)}.{nameof(HandleMessage)}");
}
}
protected override Task <DataFlowResult> Parse(DataFlowContext context)
{
if (!context.Contains(_model.TypeName))
{
context.Add(_model.TypeName, _tableMetadata);
}
var selectable = context.GetSelectable();
var results = new ParseResult <T>();
if (selectable.Properties == null)
{
selectable.Properties = new Dictionary <string, object>();
}
var environments = new Dictionary <string, string>();
foreach (var property in context.Response.Request.Properties)
{
environments.Add(property.Key, property.Value);
}
if (_model.ShareValueSelectors != null)
{
foreach (var selector in _model.ShareValueSelectors)
{
string name = selector.Name;
var value = selectable.Select(selector.ToSelector()).GetValue();
if (!environments.ContainsKey(name))
{
environments.Add(name, value);
}
else
{
environments[name] = value;
}
}
}
bool singleExtractor = _model.Selector == null;
if (!singleExtractor)
{
var selector = _model.Selector.ToSelector();
var list = selectable.SelectList(selector).Nodes()?.ToList();
if (list != null)
{
if (_model.Take > 0 && list.Count > _model.Take)
{
list = _model.TakeFromHead
? list.Take(_model.Take).ToList()
: list.Skip(list.Count - _model.Take).ToList();
}
for (var i = 0; i < list.Count; ++i)
{
var item = list.ElementAt(i);
var obj = ParseObject(environments, item, i);
if (obj != null)
{
results.Add(obj);
}
else
{
Logger?.LogWarning($"解析到空数据，类型: {_model.TypeName}");
}
}
}
}
else
{
var obj = ParseObject(environments, selectable, 0);
if (obj != null)
{
results.Add(obj);
}
else
{
Logger?.LogWarning($"解析到空数据，类型: {_model.TypeName}");
}
}
if (results.Count > 0)
{
var items = context.GetParseItem(_model.TypeName);
if (items == null)
{
context.AddParseItem(_model.TypeName, results);
}
else
{
((ParseResult <T>)items).AddRange(results);
}
}
return(Task.FromResult(DataFlowResult.Success));
}
protected override void OnWebSocketEvent(WebSocketStreamEventArgs args, IEnumerable <Action <SymbolStatisticsEventArgs> > callbacks)
{
Logger?.LogDebug($"{nameof(SymbolStatisticsWebSocketClient)}: \"{args.Json}\"");
try
{
SymbolStatisticsEventArgs eventArgs;
if (args.Json.IsJsonArray())
{
// Simulate a single event time.
var eventTime = DateTime.UtcNow.ToTimestamp().ToDateTime();
var statistics = JArray.Parse(args.Json).Select(DeserializeSymbolStatistics).ToArray();
eventArgs = new SymbolStatisticsEventArgs(eventTime, args.Token, statistics);
}
else
{
var jObject = JObject.Parse(args.Json);
var eventType = jObject["e"].Value <string>();
if (eventType == "24hrTicker")
{
var eventTime = jObject["E"].Value <long>().ToDateTime();
var statistics = DeserializeSymbolStatistics(jObject);
eventArgs = new SymbolStatisticsEventArgs(eventTime, args.Token, statistics);
}
else
{
Logger?.LogWarning($"{nameof(SymbolStatisticsWebSocketClient)}.{nameof(OnWebSocketEvent)}: Unexpected event type ({eventType}).");
return;
}
}
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
StatisticsUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsWebSocketClient)}: Unhandled aggregate trade event handler exception.");
}
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsWebSocketClient)}.{nameof(OnWebSocketEvent)}");
}
}
}
protected override async Task <Response> ImplDownloadAsync(Request request)
{
var response = new Response {
Request = request
};
for (var i = 0; i < RetryTime; ++i)
{
HttpResponseMessage httpResponseMessage = null;
WebProxy proxy = null;
try
{
var httpRequestMessage = GenerateHttpRequestMessage(request);
if (UseProxy)
{
if (HttpProxyPool == null)
{
response.Exception = "HttpProxyPool is null";
response.Success = false;
Logger?.LogError(
$"{request.OwnerId} download {request.Url} failed [{i}]: {response.Exception}");
return(response);
}
else
{
proxy = HttpProxyPool.GetProxy();
if (proxy == null)
{
response.Exception = "There is no available proxy";
response.Success = false;
Logger?.LogError(
$"{request.OwnerId} download {request.Url} failed [{i}]: {response.Exception}");
return(response);
}
}
}
var httpClientEntry = GetHttpClientEntry(proxy == null ? "DEFAULT" : $"{proxy.Address}", proxy);
httpResponseMessage = Framework.NetworkCenter == null
? await httpClientEntry.HttpClient.SendAsync(httpRequestMessage)
: await Framework.NetworkCenter.Execute(async() =>
await httpClientEntry.HttpClient.SendAsync(httpRequestMessage));
httpResponseMessage.EnsureSuccessStatusCode();
response.TargetUrl = httpResponseMessage.RequestMessage.RequestUri.AbsoluteUri;
response.MediaType = httpResponseMessage.Content.Headers.ContentType.MediaType;
response.CharSet = httpResponseMessage.Content.Headers.ContentType.CharSet;
response.Content = await httpResponseMessage.Content.ReadAsByteArrayAsync();
// if (!ExcludeMediaTypes.Any(t =>
// httpResponseMessage.Content.Headers.ContentType.MediaType.Contains(t)))
// {
// if (!DownloadFile)
// {
// StorageFile(request, bytes);
// }
// }
// else
// {
// var content = ReadContent(request, bytes,
// httpResponseMessage.Content.Headers.ContentType.CharSet);
//
// if (DecodeHtml)
// {
//#if NETFRAMEWORK
// content = System.Web.HttpUtility.UrlDecode(
// System.Web.HttpUtility.HtmlDecode(content),
// string.IsNullOrEmpty(request.Encoding)
// ? Encoding.UTF8
// : Encoding.GetEncoding(request.Encoding));
//#else
// content = WebUtility.UrlDecode(WebUtility.HtmlDecode(content));
//#endif
// }
//
// response.RawText = content;
// }
if (!string.IsNullOrWhiteSpace(request.ChangeIpPattern))
{
var rawtext = ResponseExtensions.ReadContent(request, response.Content, response.CharSet);
if (Regex.IsMatch(rawtext, request.ChangeIpPattern))
{
if (UseProxy)
{
response.TargetUrl = null;
response.Content = null;
response.Success = false;
// 把代理设置为空，影响 final 代码块里不作归还操作，等于删除此代理
proxy = null;
}
else
{
// 不支持切换 IP
if (Framework.NetworkCenter == null ||
!Framework.NetworkCenter.SupportAdsl)
{
response.Success = false;
response.Exception = "IP Banded";
Logger?.LogError(
$"{request.OwnerId} download {request.Url} failed [{i}]: {response.Exception}");
return(response);
}
else
{
Framework.NetworkCenter.Redial();
}
}
}
}
else
{
response.Success = true;
Logger?.LogInformation(
$"{request.OwnerId} download {request.Url} success");
return(response);
}
}
catch (Exception e)
{
response.Exception = e.Message;
response.Success = false;
Logger?.LogError($"{request.OwnerId} download {request.Url} failed [{i}]: {e}");
}
finally
{
if (HttpProxyPool != null && proxy != null)
{
HttpProxyPool.ReturnProxy(proxy,
httpResponseMessage?.StatusCode ?? HttpStatusCode.ServiceUnavailable);
}
try
{
httpResponseMessage?.Dispose();
}
catch (Exception e)
{
Logger?.LogWarning($"{request.OwnerId} dispose response {request.Url} failed [{i}]: {e}");
}
}
// 下载失败需要等待一秒，防止频率过高。
// TODO: 改成可配置
Thread.Sleep(1000);
}
return(response);
}
/// <summary>
/// Deserialize JSON and raise <see cref="UserDataEventArgs"/> event.
/// </summary>
/// <param name="json"></param>
/// <param name="token"></param>
/// <param name="callback"></param>
/// <returns></returns>
protected override void DeserializeJsonAndRaiseEvent(string json, CancellationToken token, Action <UserDataEventArgs> callback = null)
{
Throw.IfNullOrWhiteSpace(json, nameof(json));
Logger?.LogDebug($"{nameof(UserDataWebSocketClient)}: \"{json}\"");
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
var eventTime = jObject["E"].Value <long>();
// ReSharper disable once ConvertIfStatementToSwitchStatement
if (eventType == "outboundAccountInfo")
{
var commissions = new AccountCommissions(
jObject["m"].Value <int>(), // maker
jObject["t"].Value <int>(), // taker
jObject["b"].Value <int>(), // buyer
jObject["s"].Value <int>()); // seller
var status = new AccountStatus(
jObject["T"].Value <bool>(), // can trade
jObject["W"].Value <bool>(), // can withdraw
jObject["D"].Value <bool>()); // can deposit
var balances = jObject["B"]
.Select(entry => new AccountBalance(
entry["a"].Value <string>(), // asset
entry["f"].Value <decimal>(), // free amount
entry["l"].Value <decimal>())) // locked amount
.ToList();
var eventArgs = new AccountUpdateEventArgs(eventTime, token, new AccountInfo(User, commissions, status, jObject["u"].Value <long>(), balances));
try
{
callback?.Invoke(eventArgs);
AccountUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}: Unhandled account update event handler exception.");
}
}
}
else if (eventType == "executionReport")
{
var order = new Order(User);
FillOrder(order, jObject);
var executionType = ConvertOrderExecutionType(jObject["x"].Value <string>());
var rejectedReason = ConvertOrderRejectedReason(jObject["r"].Value <string>());
var newClientOrderId = jObject["c"].Value <string>();
if (executionType == OrderExecutionType.Trade) // trade update event.
{
var trade = new AccountTrade(
jObject["s"].Value <string>(), // symbol
jObject["t"].Value <long>(), // ID
jObject["i"].Value <long>(), // order ID
jObject["L"].Value <decimal>(), // price (price of last filled trade)
jObject["z"].Value <decimal>(), // quantity (accumulated quantity of filled trades)
jObject["n"].Value <decimal>(), // commission
jObject["N"].Value <string>(), // commission asset
jObject["T"].Value <long>(), // timestamp
order.Side == OrderSide.Buy, // is buyer
jObject["m"].Value <bool>(), // is buyer maker
jObject["M"].Value <bool>()); // is best price
var quantityOfLastFilledTrade = jObject["l"].Value <decimal>();
var eventArgs = new AccountTradeUpdateEventArgs(eventTime, token, order, rejectedReason, newClientOrderId, trade, quantityOfLastFilledTrade);
try
{
callback?.Invoke(eventArgs);
TradeUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}: Unhandled trade update event handler exception.");
}
}
}
else // order update event.
{
var eventArgs = new OrderUpdateEventArgs(eventTime, token, order, executionType, rejectedReason, newClientOrderId);
try
{
callback?.Invoke(eventArgs);
OrderUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}: Unhandled order update event handler exception.");
}
}
}
}
else
{
Logger?.LogWarning($"{nameof(UserDataWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}: Unexpected event type ({eventType}) - \"{json}\"");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(UserDataWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}");
}
}
}
public async Task <TC> FetchAsync <T, TC>(string query, object paramValues, bool paginatedQuery = false,
CommandType?dbCommandType = null, int commandTimeout = 0)
where TC : IRestmeDbEntityCollection <T>, new() where T : IRestmeDbEntity
{
try
{
SqlMapper.AddTypeMap(typeof(long), DbType.Int32);
Logger?.LogInformation($"Fetching using DB query: \n {query} ");
Logger?.LogInformation($"DB query parameters: \n {paramValues?.JsonSerialize()}");
var stopwatch = new Stopwatch();
stopwatch.Start();
var resultSet = new TC();
// if (paginatedQuery)
// {
// var results =
// await (await GetOpenConnectionAsync()).QueryMultipleAsync(query, paramValues,
// _currentTransaction,
// commandType: dbCommandType);
// var totalCount = await results.ReadSingleOrDefaultAsync<int>();
// var result = results.Read<T>().AsList();
// if (totalCount <= 0) return resultSet;
//
// resultSet.TotalRecordsCount = Convert.ToInt32(totalCount);
// if (result.Any())
// resultSet.AddRange(result);
// }
// else
// {
var results =
await(await GetOpenConnectionAsync()).QueryAsync <T>(query, paramValues, _currentTransaction,
commandType: dbCommandType, commandTimeout: commandTimeout);
var enumerable = results?.ToList();
resultSet.TotalRecordsCount = enumerable.Count;
if (paginatedQuery && enumerable?.Count > 0)
{
var firstItem = enumerable.FirstOrDefault();
if (firstItem.BaseSearchCount > 0 && firstItem.BaseSearchCount >= resultSet.TotalRecordsCount)
{
resultSet.TotalRecordsCount = firstItem.BaseSearchCount;
}
}
if (enumerable.Any())
{
resultSet.AddRange(enumerable);
}
// }
if (stopwatch.ElapsedMilliseconds >= ExecutionPerformanceThresholdInMs)
{
Logger?.LogWarning($"DB query execution time: \n {stopwatch.ElapsedMilliseconds} ms \n {query}",
dbCommandType,
query, paramValues);
}
else
{
Logger?.LogInformation(
$"DB query execution time: \n {stopwatch.ElapsedMilliseconds} ms \n {query}");
}
return(resultSet);
}
catch (Exception ex)
{
Logger?.LogError(
$"Fetching from db failed\n Query: {query}\n Error: {ex.Message}",
ex, query, paramValues, dbCommandType);
throw ex;
}
}
protected override Task <DataFlowResult> Parse(DataFlowContext context)
{
var selectable = context.Selectable;
var results = new ParseResult <T>();
if (selectable.Properties == null)
{
selectable.Properties = new Dictionary <string, object>();
}
var environments = new Dictionary <string, string>();
foreach (var property in context.Response.Request.Properties)
{
environments.Add(property.Key, property.Value);
}
if (Model.GlobalValueSelectors != null)
{
foreach (var selector in Model.GlobalValueSelectors)
{
string name = selector.Name;
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
var value = selectable.Select(selector.ToSelector()).GetValue();
if (!environments.ContainsKey(name))
{
environments.Add(name, value);
}
else
{
environments[name] = value;
}
}
}
bool singleExtractor = Model.Selector == null;
if (!singleExtractor)
{
var selector = Model.Selector.ToSelector();
var list = selectable.SelectList(selector).Nodes()?.ToList();
if (list != null)
{
if (Model.Take > 0 && list.Count > Model.Take)
{
list = Model.TakeFromHead
? list.Take(Model.Take).ToList()
: list.Skip(list.Count - Model.Take).ToList();
}
for (var i = 0; i < list.Count; ++i)
{
var item = list.ElementAt(i);
var obj = ParseObject(environments, item, i);
if (obj != null)
{
results.Add(obj);
}
else
{
Logger?.LogWarning($"解析到空数据，类型: {Model.TypeName}");
}
}
}
}
else
{
var obj = ParseObject(environments, selectable, 0);
if (obj != null)
{
results.Add(obj);
}
else
{
Logger?.LogWarning($"解析到空数据，类型: {Model.TypeName}");
}
}
AddParseResult(context, results);
return(base.Parse(context));
}
protected override void HandleMessage(IEnumerable <Action <UserDataEventArgs> > callbacks, string stream, string json)
{
if (!Users.ContainsKey(stream))
{
Logger?.LogError($"{nameof(UserDataClient)}.{nameof(HandleMessage)}: Unknown listen key (\"{stream}\"). [thread: {Thread.CurrentThread.ManagedThreadId}]");
return; // ignore.
}
var user = Users[stream];
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
var eventTime = jObject["E"].Value <long>().ToDateTime();
// ReSharper disable once ConvertIfStatementToSwitchStatement
if (eventType == "outboundAccountInfo")
{
var commissions = new AccountCommissions(
jObject["m"].Value <int>(), // maker
jObject["t"].Value <int>(), // taker
jObject["b"].Value <int>(), // buyer
jObject["s"].Value <int>()); // seller
var status = new AccountStatus(
jObject["T"].Value <bool>(), // can trade
jObject["W"].Value <bool>(), // can withdraw
jObject["D"].Value <bool>()); // can deposit
var balances = jObject["B"]
.Select(entry => new AccountBalance(
entry["a"].Value <string>(), // asset
entry["f"].Value <decimal>(), // free amount
entry["l"].Value <decimal>())) // locked amount
.ToList();
var eventArgs = new AccountUpdateEventArgs(eventTime, new AccountInfo(user, commissions, status, jObject["u"].Value <long>().ToDateTime(), balances));
try
{
// ReSharper disable once InconsistentlySynchronizedField
if (_accountUpdateSubscribers.TryGetValue(stream, out var subscribers))
{
foreach (var subcriber in subscribers)
{
subcriber(eventArgs);
}
}
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
AccountUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogWarning(e, $"{nameof(UserDataClient)}.{nameof(HandleMessage)}: Unhandled account update event handler exception.");
}
}
else if (eventType == "executionReport")
{
var order = new Order(user);
FillOrder(order, jObject);
var executionType = ConvertOrderExecutionType(jObject["x"].Value <string>());
var rejectedReason = jObject["r"].Value <string>();
var newClientOrderId = jObject["c"].Value <string>();
if (executionType == OrderExecutionType.Trade) // trade update event.
{
var trade = new AccountTrade(
jObject["s"].Value <string>(), // symbol
jObject["t"].Value <long>(), // ID
jObject["i"].Value <long>(), // order ID
jObject["L"].Value <decimal>(), // price (price of last filled trade)
jObject["z"].Value <decimal>(), // quantity (accumulated quantity of filled trades)
jObject["n"].Value <decimal>(), // commission
jObject["N"].Value <string>(), // commission asset
jObject["T"].Value <long>()
.ToDateTime(), // time
order.Side == OrderSide.Buy, // is buyer
jObject["m"].Value <bool>(), // is buyer maker
jObject["M"].Value <bool>()); // is best price
var quantityOfLastFilledTrade = jObject["l"].Value <decimal>();
var eventArgs = new AccountTradeUpdateEventArgs(eventTime, order, rejectedReason, newClientOrderId, trade, quantityOfLastFilledTrade);
try
{
// ReSharper disable once InconsistentlySynchronizedField
if (_accountTradeUpdateSubscribers.TryGetValue(stream, out var subscribers))
{
foreach (var subcriber in subscribers)
{
subcriber(eventArgs);
}
}
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
TradeUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogWarning(e, $"{nameof(UserDataClient)}.{nameof(HandleMessage)}: Unhandled trade update event handler exception.");
}
}
else // order update event.
{
var eventArgs = new OrderUpdateEventArgs(eventTime, order, executionType, rejectedReason, newClientOrderId);
try
{
// ReSharper disable once InconsistentlySynchronizedField
if (_orderUpdateSubscribers.TryGetValue(stream, out var subscribers))
{
foreach (var subcriber in subscribers)
{
subcriber(eventArgs);
}
}
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
OrderUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogWarning(e, $"{nameof(UserDataClient)}.{nameof(HandleMessage)}: Unhandled order update event handler exception.");
}
}
}
else
{
Logger?.LogWarning($"{nameof(UserDataClient)}.{nameof(HandleMessage)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(UserDataClient)}.{nameof(HandleMessage)}");
}
}
/// <summary>
/// Deserialize JSON and raise <see cref="DepthUpdateEventArgs"/> event.
/// </summary>
/// <param name="json"></param>
/// <param name="token"></param>
/// <param name="callback"></param>
protected override void DeserializeJsonAndRaiseEvent(string json, CancellationToken token, Action <DepthUpdateEventArgs> callback = null)
{
Throw.IfNullOrWhiteSpace(json, nameof(json));
Logger?.LogDebug($"{nameof(DepthWebSocketClient)}: \"{json}\"");
try
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"]?.Value <string>();
DepthUpdateEventArgs eventArgs = null;
if (eventType == null) // partial order book stream.
{
// Simulate event time.
var eventTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var lastUpdateId = jObject["lastUpdateId"].Value <long>();
var bids = jObject["bids"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToList();
var asks = jObject["asks"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToList();
eventArgs = new DepthUpdateEventArgs(eventTime, token, Symbol, lastUpdateId, lastUpdateId, bids, asks);
}
else if (eventType == "depthUpdate")
{
var symbol = jObject["s"].Value <string>();
var eventTime = jObject["E"].Value <long>();
var firstUpdateId = jObject["U"].Value <long>();
var lastUpdateId = jObject["u"].Value <long>();
var bids = jObject["b"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToList();
var asks = jObject["a"].Select(entry => (entry[0].Value <decimal>(), entry[1].Value <decimal>())).ToList();
eventArgs = new DepthUpdateEventArgs(eventTime, token, symbol, firstUpdateId, lastUpdateId, bids, asks);
}
else
{
Logger?.LogWarning($"{nameof(DepthWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}: Unexpected event type ({eventType}).");
return;
}
try
{
callback?.Invoke(eventArgs);
DepthUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DepthWebSocketClient)}: Unhandled depth update event handler exception.");
}
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(DepthWebSocketClient)}.{nameof(DeserializeJsonAndRaiseEvent)}");
}
}
}
public override async Task StreamAsync(Uri uri, CancellationToken token)
{
Throw.IfNull(uri, nameof(uri));
if (!token.CanBeCanceled)
{
throw new ArgumentException($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Token must be capable of being in the canceled state.", nameof(token));
}
if (token.IsCancellationRequested)
{
return;
}
lock (_sync)
{
if (IsStreaming)
{
throw new InvalidOperationException($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Already streaming (this method is not reentrant).");
}
IsStreaming = true;
}
var webSocket = new ClientWebSocket();
webSocket.Options.KeepAliveInterval = TimeSpan.FromSeconds(30);
try
{
try
{
Logger?.LogInformation($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Web socket connecting...");
await webSocket.ConnectAsync(uri, token)
.ConfigureAwait(false);
if (webSocket.State != WebSocketState.Open)
{
throw new Exception($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket connect failed (state: {webSocket.State}).");
}
OnOpen();
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogWarning(e, $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket connect exception (state: {webSocket.State}).");
throw;
}
}
var bytes = new byte[ReceiveBufferSize];
var buffer = new ArraySegment <byte>(bytes);
var stringBuilder = new StringBuilder();
while (!token.IsCancellationRequested)
{
stringBuilder.Clear();
try
{
WebSocketReceiveResult result;
do
{
if (webSocket.State != WebSocketState.Open && webSocket.State != WebSocketState.CloseSent)
{
break;
}
result = await webSocket
.ReceiveAsync(buffer, token)
.ConfigureAwait(false);
switch (result.MessageType)
{
case WebSocketMessageType.Close:
var message = result.CloseStatus.HasValue
? $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Web socket closed ({result.CloseStatus.Value}): \"{result.CloseStatusDescription ?? "[no reason provided]"}\""
: $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Web socket closed: \"{result.CloseStatusDescription ?? "[no reason provided]"}\"";
Logger?.LogWarning(message);
await webSocket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, string.Empty, CancellationToken.None)
.ConfigureAwait(false);
break;
case WebSocketMessageType.Text when result.Count > 0:
stringBuilder.Append(Encoding.UTF8.GetString(bytes, 0, result.Count));
break;
case WebSocketMessageType.Binary:
Logger?.LogWarning($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Received unsupported binary message type (state: {webSocket.State}).");
break;
default:
throw new ArgumentOutOfRangeException(nameof(result.MessageType), $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Unknown result message type ({result.MessageType}).");
}
}while (!result.EndOfMessage);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogWarning(e, $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket receive exception (state: {webSocket.State}).");
throw;
}
}
if (token.IsCancellationRequested || webSocket.State == WebSocketState.Aborted)
{
break;
}
var json = stringBuilder.ToString();
if (!string.IsNullOrWhiteSpace(json))
{
OnMessage(json, uri.AbsolutePath);
}
else
{
Logger?.LogWarning($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Received empty JSON message (state: {webSocket.State}).");
}
}
}
finally
{
//if (webSocket.State == WebSocketState.Open || webSocket.State == WebSocketState.CloseReceived || webSocket.State == WebSocketState.CloseSent)
//{
// try
// {
// await webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, string.Empty, CancellationToken.None)
// .ConfigureAwait(false);
// }
// catch (Exception e)
// {
// Logger?.LogWarning(e, $"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: WebSocket close exception (state: {webSocket.State}).");
// }
//}
webSocket?.Dispose();
lock (_sync) { IsStreaming = false; }
if (IsOpen)
{
OnClose();
}
Logger?.LogDebug($"{nameof(DefaultWebSocketClient)}.{nameof(StreamAsync)}: Task complete. [thread: {Thread.CurrentThread.ManagedThreadId}]");
}
}
protected override void OnWebSocketEvent(WebSocketStreamEventArgs args, IEnumerable <Action <TradeEventArgs> > callbacks)
{
Logger?.LogDebug($"{nameof(TradeWebSocketClient)}: \"{args.Json}\"");
try
{
var jObject = JObject.Parse(args.Json);
var eventType = jObject["e"].Value <string>();
if (eventType == "trade")
{
var eventTime = jObject["E"].Value <long>().ToDateTimeK();
var trade = new Trade(
jObject["s"].Value <string>(), // symbol
jObject["t"].Value <long>(), // trade ID
jObject["p"].Value <decimal>(), // price
jObject["q"].Value <decimal>(), // quantity
jObject["b"].Value <long>(), // buyer order ID
jObject["a"].Value <long>(), // seller order ID
jObject["T"].Value <long>()
.ToDateTimeK(), // trade time
jObject["m"].Value <bool>(), // is buyer the market maker?
jObject["M"].Value <bool>()); // is best price match?
var eventArgs = new TradeEventArgs(eventTime, args.Token, trade);
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
Trade?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(TradeWebSocketClient)}: Unhandled aggregate trade event handler exception.");
}
}
}
else
{
Logger?.LogWarning($"{nameof(TradeWebSocketClient)}.{nameof(OnWebSocketEvent)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(TradeWebSocketClient)}.{nameof(OnWebSocketEvent)}");
}
}
}
/// <summary>
/// Tries to fetch the result from disk cache, the memory queue, or create it. If the memory queue has space,
/// the writeCallback() will be executed and the resulting bytes put in a queue for writing to disk.
/// If the memory queue is full, writing to disk will be attempted synchronously.
/// In either case, writing to disk can also fail if the disk cache is full and eviction fails.
/// If the memory queue is full, eviction will be done synchronously and can cause other threads to time out
/// while waiting for QueueLock
/// </summary>
/// <param name="key"></param>
/// <param name="dataProviderCallback"></param>
/// <param name="cancellationToken"></param>
/// <param name="retrieveContentType"></param>
/// <returns></returns>
/// <exception cref="OperationCanceledException"></exception>
/// <exception cref="ArgumentOutOfRangeException"></exception>
public async Task <AsyncCacheResult> GetOrCreateBytes(
byte[] key,
AsyncBytesResult dataProviderCallback,
CancellationToken cancellationToken,
bool retrieveContentType)
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException(cancellationToken);
}
var swGetOrCreateBytes = Stopwatch.StartNew();
var entry = new CacheEntry(key, PathBuilder);
// Tell cleanup what we're using
CleanupManager.NotifyUsed(entry);
// Fast path on disk hit
var swFileExists = Stopwatch.StartNew();
var fileBasedResult = await TryGetFileBasedResult(entry, false, retrieveContentType, cancellationToken);
if (fileBasedResult != null)
{
return(fileBasedResult);
}
// Just continue on creating the file. It must have been deleted between the calls
swFileExists.Stop();
var cacheResult = new AsyncCacheResult();
//Looks like a miss. Let's enter a lock for the creation of the file. This is a different locking system
// than for writing to the file
//This prevents two identical requests from duplicating efforts. Different requests don't lock.
//Lock execution using relativePath as the sync basis. Ignore casing differences. This prevents duplicate entries in the write queue and wasted CPU/RAM usage.
var queueLockComplete = await QueueLocks.TryExecuteAsync(entry.StringKey,
Options.WaitForIdenticalRequestsTimeoutMs, cancellationToken,
async() =>
{
var swInsideQueueLock = Stopwatch.StartNew();
// Now, if the item we seek is in the queue, we have a memcached hit.
// If not, we should check the filesystem. It's possible the item has been written to disk already.
// If both are a miss, we should see if there is enough room in the write queue.
// If not, switch to in-thread writing.
var existingQueuedWrite = CurrentWrites.Get(entry.StringKey);
if (existingQueuedWrite != null)
{
cacheResult.Data = existingQueuedWrite.GetReadonlyStream();
cacheResult.ContentType = existingQueuedWrite.ContentType;
cacheResult.Detail = AsyncCacheDetailResult.MemoryHit;
return;
}
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException(cancellationToken);
}
swFileExists.Start();
// Fast path on disk hit, now that we're in a synchronized state
var fileBasedResult2 = await TryGetFileBasedResult(entry, true, retrieveContentType, cancellationToken);
if (fileBasedResult2 != null)
{
cacheResult = fileBasedResult2;
return;
}
// Just continue on creating the file. It must have been deleted between the calls
swFileExists.Stop();
var swDataCreation = Stopwatch.StartNew();
//Read, resize, process, and encode the image. Lots of exceptions thrown here.
var result = await dataProviderCallback(cancellationToken);
swDataCreation.Stop();
//Create AsyncWrite object to enqueue
var w = new AsyncWrite(entry.StringKey, result.Item2, result.Item1);
cacheResult.Detail = AsyncCacheDetailResult.Miss;
cacheResult.ContentType = w.ContentType;
cacheResult.Data = w.GetReadonlyStream();
// Create a lambda which we can call either in a spawned Task (if enqueued successfully), or
// in this task, if our buffer is full.
async Task <AsyncCacheDetailResult> EvictWriteAndLogUnsynchronized(bool queueFull, TimeSpan dataCreationTime, CancellationToken ct)
{
var delegateStartedAt = DateTime.UtcNow;
var swReserveSpace = Stopwatch.StartNew();
//We only permit eviction proceedings from within the queue or if the queue is disabled
var allowEviction = !queueFull || CurrentWrites.MaxQueueBytes <= 0;
var reserveSpaceResult = await CleanupManager.TryReserveSpace(entry, w.ContentType,
w.GetUsedBytes(), allowEviction, EvictAndWriteLocks, ct);
swReserveSpace.Stop();
var syncString = queueFull ? "synchronous" : "async";
if (!reserveSpaceResult.Success)
{
Logger?.LogError(
queueFull
? "HybridCache synchronous eviction failed; {Message}. Time taken: {1}ms - {2}"
: "HybridCache async eviction failed; {Message}. Time taken: {1}ms - {2}",
syncString, reserveSpaceResult.Message, swReserveSpace.ElapsedMilliseconds,
entry.RelativePath);
return(AsyncCacheDetailResult.CacheEvictionFailed);
}
var swIo = Stopwatch.StartNew();
// We only force an immediate File.Exists check when running from the Queue
// Otherwise it happens inside the lock
var fileWriteResult = await FileWriter.TryWriteFile(entry, delegate(Stream s, CancellationToken ct2)
{
if (ct2.IsCancellationRequested)
{
throw new OperationCanceledException(ct2);
}
var fromStream = w.GetReadonlyStream();
return(fromStream.CopyToAsync(s, 81920, ct2));
}, !queueFull, Options.WaitForIdenticalDiskWritesMs, ct);
swIo.Stop();
var swMarkCreated = Stopwatch.StartNew();
// Mark the file as created so it can be deleted
await CleanupManager.MarkFileCreated(entry,
w.ContentType,
w.GetUsedBytes(),
DateTime.UtcNow);
swMarkCreated.Stop();
switch (fileWriteResult)
{
case CacheFileWriter.FileWriteStatus.LockTimeout:
//We failed to lock the file.
Logger?.LogWarning("HybridCache {Sync} write failed; disk lock timeout exceeded after {IoTime}ms - {Path}",
syncString, swIo.ElapsedMilliseconds, entry.RelativePath);
return(AsyncCacheDetailResult.WriteTimedOut);
case CacheFileWriter.FileWriteStatus.FileAlreadyExists:
Logger?.LogTrace("HybridCache {Sync} write found file already exists in {IoTime}ms, after a {DelayTime}ms delay and {CreationTime}- {Path}",
syncString, swIo.ElapsedMilliseconds,
delegateStartedAt.Subtract(w.JobCreatedAt).TotalMilliseconds,
dataCreationTime, entry.RelativePath);
return(AsyncCacheDetailResult.FileAlreadyExists);
case CacheFileWriter.FileWriteStatus.FileCreated:
if (queueFull)
{
Logger?.LogTrace(@"HybridCache synchronous write complete. Create: {CreateTime}ms. Write {WriteTime}ms. Mark Created: {MarkCreatedTime}ms. Eviction: {EvictionTime}ms - {Path}",
Math.Round(dataCreationTime.TotalMilliseconds).ToString(CultureInfo.InvariantCulture).PadLeft(4),
swIo.ElapsedMilliseconds.ToString().PadLeft(4),
swMarkCreated.ElapsedMilliseconds.ToString().PadLeft(4),
swReserveSpace.ElapsedMilliseconds.ToString().PadLeft(4), entry.RelativePath);
}
else
{
Logger?.LogTrace(@"HybridCache async write complete. Create: {CreateTime}ms. Write {WriteTime}ms. Mark Created: {MarkCreatedTime}ms Eviction {EvictionTime}ms. Delay {DelayTime}ms. - {Path}",
Math.Round(dataCreationTime.TotalMilliseconds).ToString(CultureInfo.InvariantCulture).PadLeft(4),
swIo.ElapsedMilliseconds.ToString().PadLeft(4),
swMarkCreated.ElapsedMilliseconds.ToString().PadLeft(4),
swReserveSpace.ElapsedMilliseconds.ToString().PadLeft(4),
Math.Round(delegateStartedAt.Subtract(w.JobCreatedAt).TotalMilliseconds).ToString(CultureInfo.InvariantCulture).PadLeft(4),
entry.RelativePath);
}
return(AsyncCacheDetailResult.WriteSucceeded);
default:
throw new ArgumentOutOfRangeException();
}
}
async Task <AsyncCacheDetailResult> EvictWriteAndLogSynchronized(bool queueFull,
TimeSpan dataCreationTime, CancellationToken ct)
{
var cacheDetailResult = AsyncCacheDetailResult.Unknown;
var writeLockComplete = await EvictAndWriteLocks.TryExecuteAsync(entry.StringKey,
Options.WaitForIdenticalRequestsTimeoutMs, cancellationToken,
async() =>
{
cacheDetailResult =
await EvictWriteAndLogUnsynchronized(queueFull, dataCreationTime, ct);
});
if (!writeLockComplete)
{
cacheDetailResult = AsyncCacheDetailResult.EvictAndWriteLockTimedOut;
}
return(cacheDetailResult);
}
var swEnqueue = Stopwatch.StartNew();
var queueResult = CurrentWrites.Queue(w, async delegate
{
try
{
var unused = await EvictWriteAndLogSynchronized(false, swDataCreation.Elapsed, CancellationToken.None);
}
catch (Exception ex)
{
Logger?.LogError(ex, "HybridCache failed to flush async write, {Exception} {Path}\n{StackTrace}", ex.ToString(),
entry.RelativePath, ex.StackTrace);
}
});
swEnqueue.Stop();
swInsideQueueLock.Stop();
swGetOrCreateBytes.Stop();
if (queueResult == AsyncWriteCollection.AsyncQueueResult.QueueFull)
{
if (Options.WriteSynchronouslyWhenQueueFull)
{
var writerDelegateResult = await EvictWriteAndLogSynchronized(true, swDataCreation.Elapsed, cancellationToken);
cacheResult.Detail = writerDelegateResult;
}
}
});
public override async Task StreamAsync(Uri uri, CancellationToken token)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
if (!token.CanBeCanceled)
{
throw new ArgumentException("Token must be capable of being in the canceled state.", nameof(token));
}
token.ThrowIfCancellationRequested();
Exception exception = null;
var tcs = new TaskCompletionSource <object>();
token.Register(() => tcs.TrySetCanceled());
var webSocket = new WebSocket4Net.WebSocket(uri.AbsoluteUri);
webSocket.Opened += (s, e) =>
{
IsStreaming = true;
RaiseOpenEvent();
};
webSocket.Closed += (s, e) => tcs.TrySetCanceled();
webSocket.MessageReceived += (s, evt) =>
{
try
{
var json = evt.Message;
if (!string.IsNullOrWhiteSpace(json))
{
RaiseMessageEvent(new WebSocketClientEventArgs(json));
}
else
{
Logger?.LogWarning($"{nameof(WebSocket4NetClient)}.MessageReceived: Received empty JSON message.");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(WebSocket4NetClient)}.MessageReceived: WebSocket read exception.");
exception = e;
tcs.TrySetCanceled();
}
}
};
webSocket.Error += (s, e) =>
{
if (token.IsCancellationRequested)
{
return;
}
Logger?.LogError(e.Exception, $"{nameof(WebSocket4NetClient)}.Error: WebSocket exception.");
exception = e.Exception;
tcs.TrySetCanceled();
};
try
{
webSocket.Open();
await tcs.Task
.ConfigureAwait(false);
if (exception != null)
{
throw exception;
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(WebSocket4NetClient)}.{nameof(StreamAsync)}: WebSocket open exception.");
throw;
}
}
finally
{
if (webSocket.State == WebSocket4Net.WebSocketState.Open)
{
try { webSocket.Close(); }
catch (Exception e)
{
Logger?.LogError(e, $"{nameof(WebSocket4NetClient)}.{nameof(StreamAsync)}: WebSocket close exception.");
}
}
webSocket.Dispose();
if (IsStreaming)
{
IsStreaming = false;
RaiseCloseEvent();
}
}
}
protected override Task HandleMessageAsync(IEnumerable <Action <SymbolStatisticsEventArgs> > callbacks, string stream, string json, CancellationToken token = default)
{
try
{
SymbolStatisticsEventArgs eventArgs;
if (json.IsJsonArray())
{
// Simulate a single event time.
var eventTime = DateTime.UtcNow.ToTimestamp().ToDateTime();
var statistics = JArray.Parse(json).Select(DeserializeSymbolStatistics).ToArray();
eventArgs = new SymbolStatisticsEventArgs(eventTime, token, statistics);
}
else
{
var jObject = JObject.Parse(json);
var eventType = jObject["e"].Value <string>();
if (eventType == "24hrTicker")
{
var eventTime = jObject["E"].Value <long>().ToDateTime();
var statistics = DeserializeSymbolStatistics(jObject);
eventArgs = new SymbolStatisticsEventArgs(eventTime, token, statistics);
}
else
{
Logger?.LogWarning($"{nameof(SymbolStatisticsClient)}.{nameof(HandleMessageAsync)}: Unexpected event type ({eventType}).");
return(Task.CompletedTask);
}
}
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
StatisticsUpdate?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsClient)}: Unhandled aggregate trade event handler exception.");
}
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(SymbolStatisticsClient)}.{nameof(HandleMessageAsync)}");
}
}
return(Task.CompletedTask);
}
protected override void OnWebSocketEvent(WebSocketStreamEventArgs args, IEnumerable <Action <CandlestickEventArgs> > callbacks)
{
Logger?.LogDebug($"{nameof(CandlestickWebSocketClient)}: \"{args.Json}\"");
try
{
var jObject = JObject.Parse(args.Json);
var eventType = jObject["e"].Value <string>();
if (eventType == "kline")
{
//var symbol = jObject["s"].Value<string>();
var eventTime = jObject["E"].Value <long>();
var firstTradeId = jObject["k"]["f"].Value <long>();
var lastTradeId = jObject["k"]["L"].Value <long>();
var isFinal = jObject["k"]["x"].Value <bool>();
var candlestick = new Candlestick(
jObject["k"]["s"].Value <string>(), // symbol
jObject["k"]["i"].Value <string>()
.ToCandlestickInterval(), // interval
jObject["k"]["t"].Value <long>(), // open time
jObject["k"]["o"].Value <decimal>(), // open
jObject["k"]["h"].Value <decimal>(), // high
jObject["k"]["l"].Value <decimal>(), // low
jObject["k"]["c"].Value <decimal>(), // close
jObject["k"]["v"].Value <decimal>(), // volume
jObject["k"]["T"].Value <long>(), // close time
jObject["k"]["q"].Value <decimal>(), // quote asset volume
jObject["k"]["n"].Value <long>(), // number of trades
jObject["k"]["V"].Value <decimal>(), // taker buy base asset volume (volume of active buy)
jObject["k"]["Q"].Value <decimal>() // taker buy quote asset volume (quote volume of active buy)
);
var eventArgs = new CandlestickEventArgs(eventTime, args.Token, candlestick, firstTradeId, lastTradeId, isFinal);
try
{
if (callbacks != null)
{
foreach (var callback in callbacks)
{
callback(eventArgs);
}
}
Candlestick?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(CandlestickWebSocketClient)}: Unhandled candlestick event handler exception.");
}
}
}
else
{
Logger?.LogWarning($"{nameof(CandlestickWebSocketClient)}.{nameof(OnWebSocketEvent)}: Unexpected event type ({eventType}).");
}
}
catch (OperationCanceledException) { }
catch (Exception e)
{
if (!args.Token.IsCancellationRequested)
{
Logger?.LogError(e, $"{nameof(CandlestickWebSocketClient)}.{nameof(OnWebSocketEvent)}");
}
}
}
