private void RemoveFromCorrelationMatching(AsyncItem asyncItem, Exception exception = null) { if (asyncItem == null) { return; } var correlationId = asyncItem.Context.CorrelationId; AsyncItem request; if (_requestsByCorrelation.TryRemove(correlationId, out request)) { _log.Info(() => LogEvent.Create($"Removed request {request.ApiKey} (id {correlationId}): timed out or otherwise errored in client.")); if (_timedOutRequestsByCorrelation.Count > 100) { _log.Debug(() => LogEvent.Create($"Clearing timed out requests to avoid overflow ({_timedOutRequestsByCorrelation.Count}).")); _timedOutRequestsByCorrelation.Clear(); } _timedOutRequestsByCorrelation.TryAdd(correlationId, request); } if (_disposeToken.IsCancellationRequested) { asyncItem.ReceiveTask.TrySetException(new ObjectDisposedException("The object is being disposed and the connection is closing.")); } else if (exception != null) { asyncItem.ReceiveTask.TrySetException(exception); } else { asyncItem.ReceiveTask.TrySetException(new TimeoutException($"Timeout expired after {_configuration.RequestTimeout.TotalMilliseconds} ms.")); } }
public async Task TrueAwaitsAreSupported() { const int expectedValue = 42; TaskCompletionSource <int> tcs = new TaskCompletionSource <int>(); IAsyncEnumerator <int> enumerator = AwaitTaskThenYieldItem(); AsyncItem <int> nextItemTask = enumerator.GetNextAsync(); nextItemTask.IsCompleted.Should().BeFalse(); tcs.SetResult(expectedValue); Optional <int> yieldedItem = await nextItemTask.WithSyncContext(); yieldedItem.HasValue.Should().BeTrue(); yieldedItem.Value.Should().Be(expectedValue); async IAsyncEnumerator <int> AwaitTaskThenYieldItem() { int awaitedValue = await tcs.Task.ConfigureAwait(false); await AsyncYield.Item(awaitedValue); return(await AsyncYield.Break <int>()); } }
public async Task AsyncExceptionIsPropagatedToGetNextCall() { TaskCompletionSource <int> tcs = new TaskCompletionSource <int>(); IAsyncEnumerator <int> enumerator = AwaitAndYieldItem(); AsyncItem <int> itemTask = enumerator.GetNextAsync(); tcs.SetException(new Exception("The punch has been spiked")); Func <Task <Optional <int> > > awaitFaultyItemAct = async() => await itemTask; awaitFaultyItemAct.ShouldThrow <Exception>(); Optional <int> itemAfterException = await enumerator.GetNextAsync().WithSyncContext(); itemAfterException.HasValue.Should().BeFalse(); async IAsyncEnumerator <int> AwaitAndYieldItem() { int awaitedItem = await tcs.Task.ConfigureAwait(false); await AsyncYield.Item(awaitedItem); return(await AsyncYield.Break <int>()); } }
/// <summary> /// 异步任务 /// </summary> private static IEnumerator AsyncTask() { int emptyTaskCount = 0; int endCount = 10; while (true) { if (_asyncTaskItems.Count > 0) { emptyTaskCount = 0; AsyncItem item = _asyncTaskItems.Dequeue(); if (item != null) { Load(item.key, item.customLoader, item.callback, item.cacheLevel); } } else { if (++emptyTaskCount >= endCount) { break; } } yield return(null); } _asyncTask = null; }
private void TriggerError(AsyncItem asyncItem, Exception exception) { var action = this.AsyncError; if (action != null) { action(asyncItem.writer, new LogExceptionEventArgs(exception)); } }
static int SetItemsCounter(IntPtr L) { LuaScriptMgr.CheckArgsCount(L, 2); AsyncItem obj = (AsyncItem)LuaScriptMgr.GetUnityObjectSelf(L, 1, "AsyncItem"); int arg0 = (int)LuaScriptMgr.GetNumber(L, 2); obj.SetItemsCounter(arg0); return(0); }
//重载将数据进行异常托管 protected override void Flush(byte[] buffer) { var asyncItem = new AsyncItem(); asyncItem.buffer = buffer; asyncItem.filepath = this.BuildPath(); asyncItem.writer = this; this.asyncQueue.Add(asyncItem); }
private void AddToCorrelationMatching(AsyncItem asyncItem) { if (asyncItem == null) { return; } if (_requestsByCorrelation.TryAdd(asyncItem.Context.CorrelationId, asyncItem) == false) { throw new KafkaException("Failed to register request for async response."); } asyncItem.OnTimedOut(RemoveFromCorrelationMatching, _configuration.RequestTimeout); }
/// <summary> /// Send kafka payload to server and receive a task event when response is received. /// </summary> /// <typeparam name="T">A Kafka response object return by decode function.</typeparam> /// <param name="request">The IRequest to send to the kafka servers.</param> /// <param name="context">The context for the request.</param> /// <param name="cancellationToken">Cancellation token used to cancel the transfer.</param> /// <returns></returns> public async Task <T> SendAsync <T>(IRequest <T> request, CancellationToken cancellationToken, IRequestContext context = null) where T : class, IResponse { var version = context?.ApiVersion; if (!version.HasValue) { version = await GetVersionAsync(request.ApiKey, cancellationToken).ConfigureAwait(false); } context = RequestContext.Copy(context, NextCorrelationId(), version, encoders: _configuration.Encoders, onProduceRequestMessages: _configuration.OnProduceRequestMessages); ArraySegment <byte> receivedBytes; using (var asyncItem = new AsyncItem(context, request)) { if (request.ExpectResponse) { AddToCorrelationMatching(asyncItem); } using (var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _disposeToken.Token)) { var timer = new Stopwatch(); try { await _transport.ConnectAsync(cancellation.Token).ConfigureAwait(false); var item = asyncItem; _log.Info(() => LogEvent.Create($"Sending {request.ShortString()} (id {context.CorrelationId}, {item.RequestBytes.Count} bytes) to {Endpoint}")); _log.Debug(() => LogEvent.Create($"{request.ApiKey} -----> {Endpoint} {{Context:{context},\nRequest:{request}}}")); _configuration.OnWriting?.Invoke(Endpoint, request.ApiKey); timer.Start(); var bytesWritten = await _transport.WriteBytesAsync(asyncItem.RequestBytes, cancellation.Token, context.CorrelationId).ConfigureAwait(false); timer.Stop(); _configuration.OnWritten?.Invoke(Endpoint, request.ApiKey, bytesWritten, timer.Elapsed); if (!request.ExpectResponse) { return(default(T)); } } catch (Exception ex) { timer.Stop(); _configuration.OnWriteFailed?.Invoke(Endpoint, request.ApiKey, timer.Elapsed, ex); RemoveFromCorrelationMatching(asyncItem, ex); throw; } receivedBytes = await asyncItem.ReceiveTask.Task.ThrowIfCancellationRequested(cancellation.Token).ConfigureAwait(false); } } var response = request.ToResponse(context, receivedBytes); _log.Debug(() => LogEvent.Create($"{Endpoint} -----> {{Context:{context},\n{request.ApiKey}Response:{response}}}")); return(response); }
/// <summary> /// The bulk processor. /// </summary> /// <param name="item"><see cref="TInput"/> to process</param> /// <param name="cancellationToken"><see cref="CancellationToken"/></param> /// <returns><see cref="Task"/></returns> protected override async Task Process(AsyncItem <TInput, TOutput> item, CancellationToken cancellationToken) { var policy = Policy .Handle <Exception>(ex => !(ex is TaskCanceledException || ex is OperationCanceledException)) .WaitAndRetryAsync(_clusterOptions.RetryAttempt, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (exception, sleepDuration, retry, context) => { if (retry >= _clusterOptions.RetryAttempt) { _logger.LogError( $"Could not process item after {retry} retry times: {exception.Message}"); item.TaskCompletionSource.TrySetException(exception); } }); var policyResult = await policy.ExecuteAndCaptureAsync(async ct => { try { if (CpuUsage > _clusterOptions.LimitCpuUsage) { var suspensionTime = (CpuUsage - _clusterOptions.LimitCpuUsage) / CpuUsage * 100; await Task.Delay((int)suspensionTime, ct); } var result = await _remoteContract.ProcessRemotely(item.Item, NodeMetrics); item.TaskCompletionSource.TrySetResult(result); } catch (Exception ex) when(ex is TaskCanceledException || ex is OperationCanceledException) { _logger.LogTrace("The item process has been cancelled."); item.TaskCompletionSource.TrySetCanceled(); } finally { Interlocked.Increment(ref _processedItems); } }, cancellationToken).ConfigureAwait(false); if (policyResult.Outcome == OutcomeType.Failure) { item.TaskCompletionSource.TrySetException(policyResult.FinalException); _logger.LogCritical( policyResult.FinalException != null ? $"Could not process item: {policyResult.FinalException.Message}." : "An error has occured while processing the item."); } }
static int set_OnCreateItem(IntPtr L) { object o = LuaScriptMgr.GetLuaObject(L, 1); AsyncItem obj = (AsyncItem)o; if (obj == null) { LuaTypes types = LuaDLL.lua_type(L, 1); if (types == LuaTypes.LUA_TTABLE) { LuaDLL.luaL_error(L, "unknown member name OnCreateItem"); } else { LuaDLL.luaL_error(L, "attempt to index OnCreateItem on a nil value"); } } LuaTypes funcType = LuaDLL.lua_type(L, 3); if (funcType != LuaTypes.LUA_TFUNCTION) { obj.OnCreateItem = (Func <int, Transform, GameObject>)LuaScriptMgr.GetNetObject(L, 3, typeof(Func <int, Transform, GameObject>)); } else { LuaFunction func = LuaScriptMgr.ToLuaFunction(L, 3); obj.OnCreateItem = (param0, param1) => { int top = func.BeginPCall(); LuaScriptMgr.Push(L, param0); LuaScriptMgr.Push(L, param1); func.PCall(top, 2); object[] objs = func.PopValues(top); func.EndPCall(top); return((GameObject)objs[0]); }; } return(0); }
private void AsyncLogFlush(AsyncItem asyncItem, bool retry) { try { using (var fs = new FileStream(asyncItem.filepath, FileMode.Append, FileAccess.Write)) { fs.Write(asyncItem.buffer, 0, asyncItem.buffer.Length); } } catch (DirectoryNotFoundException) { //已为重试 if (retry) { //backgroup save ignore error return; } //igore runing of remove rootpath try { var dir = System.IO.Path.GetDirectoryName(asyncItem.filepath); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } } catch (Exception exception) { this.TriggerError(asyncItem, exception); } this.AsyncLogFlush(asyncItem, true); } catch (Exception exception) { //backgroup save ignore error this.TriggerError(asyncItem, exception); } }
static int get_OnCreateItem(IntPtr L) { object o = LuaScriptMgr.GetLuaObject(L, 1); AsyncItem obj = (AsyncItem)o; if (obj == null) { LuaTypes types = LuaDLL.lua_type(L, 1); if (types == LuaTypes.LUA_TTABLE) { LuaDLL.luaL_error(L, "unknown member name OnCreateItem"); } else { LuaDLL.luaL_error(L, "attempt to index OnCreateItem on a nil value"); } } LuaScriptMgr.Push(L, obj.OnCreateItem); return(1); }
public void GetNextThrowsIfCalledBeforePreviousGetNextCompletes() { TaskCompletionSource <int>[] taskSources = Enumerable.Range(0, 2).Select(_ => new TaskCompletionSource <int>()).ToArray(); IAsyncEnumerator <int> enumerator = TaskDependentEnumerator(); AsyncItem <int> firstItemTask = enumerator.GetNextAsync(); Func <Task <Optional <int> > > despicableAct = async() => await enumerator.GetNextAsync(); despicableAct.ShouldThrow <PreviousItemNotCompletedException>(); async IAsyncEnumerator <int> TaskDependentEnumerator() { foreach (TaskCompletionSource <int> tcs in taskSources) { int item = await tcs.Task.ConfigureAwait(false); await AsyncYield.Item(item); } return(await AsyncYield.Break <int>()); } }
private void AsyncLogFlush(AsyncItem asyncItem) { this.AsyncLogFlush(asyncItem, false); }
private async Task DedicatedReceiveAsync() { // only allow one reader to execute, dump out all other requests if (Interlocked.Increment(ref _receiveCount) > 1) { return; } try { var header = new byte[Request.IntegerByteSize + Request.CorrelationSize]; AsyncItem asyncItem = null; while (!_disposeToken.IsCancellationRequested) { await Retry // use backoff so we don't take over the CPU when there's a failure .WithBackoff(int.MaxValue, minimumDelay : TimeSpan.FromMilliseconds(5), maximumDelay : TimeSpan.FromSeconds(5)) .TryAsync( async(retryAttempt, elapsed) => { await _transport.ConnectAsync(_disposeToken.Token).ConfigureAwait(false); // reading the header if (asyncItem == null) { await _transport.ReadBytesAsync(new ArraySegment <byte>(header), _disposeToken.Token).ConfigureAwait(false); var responseSize = BitConverter.ToInt32(header, 0).ToBigEndian(); var correlationId = BitConverter.ToInt32(header, Request.IntegerByteSize).ToBigEndian(); asyncItem = LookupByCorrelateId(correlationId, responseSize); if (asyncItem.ResponseBytes.Count > 0) { _log.Error(LogEvent.Create($"Request id {correlationId} matched a previous response ({asyncItem.ResponseBytes.Count + Request.CorrelationSize} of {responseSize + Request.CorrelationSize} bytes), now overwriting with {responseSize}? bytes")); } asyncItem.ResponseBytes = new ArraySegment <byte>(new byte[responseSize - Request.CorrelationSize]); } // reading the body await _transport.ReadBytesAsync(asyncItem.ResponseBytes, CancellationToken.None).ConfigureAwait(false); asyncItem.ResponseCompleted(_log); asyncItem = null; return(new RetryAttempt <bool>(true)); }, (exception, retryAttempt, retryDelay) => { if (_disposeToken.IsCancellationRequested) { exception.PrepareForRethrow(); _disposeToken.Token.ThrowIfCancellationRequested(); // in case exception is null } if (exception is ConnectionException) { _log.Warn(() => LogEvent.Create(exception, "Socket has been re-established, so recovery of the remaining of the current message is uncertain at best")); try { asyncItem.ResponseCompleted(_log); } catch { // ignore } asyncItem = null; } _log.Write(retryAttempt == 0 ? LogLevel.Error : LogLevel.Info, () => LogEvent.Create(exception, $"Failed receiving from {Endpoint} on retry {retryAttempt}: Will retry in {retryDelay}")); }, _disposeToken.Token).ConfigureAwait(false); } } catch (Exception ex) { _log.Debug(() => LogEvent.Create(ex)); } finally { await DisposeAsync().ConfigureAwait(false); Interlocked.Decrement(ref _receiveCount); _log.Info(() => LogEvent.Create($"Stopped receiving from {Endpoint}")); } }
public void AddAsyncOperation(AsyncItem async) { m_AsyncList.AddLast(async); }