private void RemoveFromCorrelationMatching(AsyncItem asyncItem, Exception exception = null)
{
if (asyncItem == null)
{
return;
}
var correlationId = asyncItem.Context.CorrelationId;
AsyncItem request;
if (_requestsByCorrelation.TryRemove(correlationId, out request))
{
_log.Info(() => LogEvent.Create($"Removed request {request.ApiKey} (id {correlationId}): timed out or otherwise errored in client."));
if (_timedOutRequestsByCorrelation.Count > 100)
{
_log.Debug(() => LogEvent.Create($"Clearing timed out requests to avoid overflow ({_timedOutRequestsByCorrelation.Count})."));
_timedOutRequestsByCorrelation.Clear();
}
_timedOutRequestsByCorrelation.TryAdd(correlationId, request);
}
if (_disposeToken.IsCancellationRequested)
{
asyncItem.ReceiveTask.TrySetException(new ObjectDisposedException("The object is being disposed and the connection is closing."));
}
else if (exception != null)
{
asyncItem.ReceiveTask.TrySetException(exception);
}
else
{
asyncItem.ReceiveTask.TrySetException(new TimeoutException($"Timeout expired after {_configuration.RequestTimeout.TotalMilliseconds} ms."));
}
}
public async Task TrueAwaitsAreSupported()
{
const int expectedValue = 42;
TaskCompletionSource <int> tcs = new TaskCompletionSource <int>();
IAsyncEnumerator <int> enumerator = AwaitTaskThenYieldItem();
AsyncItem <int> nextItemTask = enumerator.GetNextAsync();
nextItemTask.IsCompleted.Should().BeFalse();
tcs.SetResult(expectedValue);
Optional <int> yieldedItem = await nextItemTask.WithSyncContext();
yieldedItem.HasValue.Should().BeTrue();
yieldedItem.Value.Should().Be(expectedValue);
async IAsyncEnumerator <int> AwaitTaskThenYieldItem()
{
int awaitedValue = await tcs.Task.ConfigureAwait(false);
await AsyncYield.Item(awaitedValue);
return(await AsyncYield.Break <int>());
}
}
public async Task AsyncExceptionIsPropagatedToGetNextCall()
{
TaskCompletionSource <int> tcs = new TaskCompletionSource <int>();
IAsyncEnumerator <int> enumerator = AwaitAndYieldItem();
AsyncItem <int> itemTask = enumerator.GetNextAsync();
tcs.SetException(new Exception("The punch has been spiked"));
Func <Task <Optional <int> > > awaitFaultyItemAct = async() => await itemTask;
awaitFaultyItemAct.ShouldThrow <Exception>();
Optional <int> itemAfterException = await enumerator.GetNextAsync().WithSyncContext();
itemAfterException.HasValue.Should().BeFalse();
async IAsyncEnumerator <int> AwaitAndYieldItem()
{
int awaitedItem = await tcs.Task.ConfigureAwait(false);
await AsyncYield.Item(awaitedItem);
return(await AsyncYield.Break <int>());
}
}
/// <summary>
/// 异步任务
/// </summary>
private static IEnumerator AsyncTask()
{
int emptyTaskCount = 0;
int endCount = 10;
while (true)
{
if (_asyncTaskItems.Count > 0)
{
emptyTaskCount = 0;
AsyncItem item = _asyncTaskItems.Dequeue();
if (item != null)
{
Load(item.key, item.customLoader, item.callback, item.cacheLevel);
}
}
else
{
if (++emptyTaskCount >= endCount)
{
break;
}
}
yield return(null);
}
_asyncTask = null;
}
private void TriggerError(AsyncItem asyncItem, Exception exception)
{
var action = this.AsyncError;
if (action != null)
{
action(asyncItem.writer, new LogExceptionEventArgs(exception));
}
}
static int SetItemsCounter(IntPtr L)
{
LuaScriptMgr.CheckArgsCount(L, 2);
AsyncItem obj = (AsyncItem)LuaScriptMgr.GetUnityObjectSelf(L, 1, "AsyncItem");
int arg0 = (int)LuaScriptMgr.GetNumber(L, 2);
obj.SetItemsCounter(arg0);
return(0);
}
//重载将数据进行异常托管
protected override void Flush(byte[] buffer)
{
var asyncItem = new AsyncItem();
asyncItem.buffer = buffer;
asyncItem.filepath = this.BuildPath();
asyncItem.writer = this;
this.asyncQueue.Add(asyncItem);
}
private void AddToCorrelationMatching(AsyncItem asyncItem)
{
if (asyncItem == null)
{
return;
}
if (_requestsByCorrelation.TryAdd(asyncItem.Context.CorrelationId, asyncItem) == false)
{
throw new KafkaException("Failed to register request for async response.");
}
asyncItem.OnTimedOut(RemoveFromCorrelationMatching, _configuration.RequestTimeout);
}
/// <summary>
/// Send kafka payload to server and receive a task event when response is received.
/// </summary>
/// <typeparam name="T">A Kafka response object return by decode function.</typeparam>
/// <param name="request">The IRequest to send to the kafka servers.</param>
/// <param name="context">The context for the request.</param>
/// <param name="cancellationToken">Cancellation token used to cancel the transfer.</param>
/// <returns></returns>
public async Task <T> SendAsync <T>(IRequest <T> request, CancellationToken cancellationToken, IRequestContext context = null) where T : class, IResponse
{
var version = context?.ApiVersion;
if (!version.HasValue)
{
version = await GetVersionAsync(request.ApiKey, cancellationToken).ConfigureAwait(false);
}
context = RequestContext.Copy(context, NextCorrelationId(), version, encoders: _configuration.Encoders, onProduceRequestMessages: _configuration.OnProduceRequestMessages);
ArraySegment <byte> receivedBytes;
using (var asyncItem = new AsyncItem(context, request)) {
if (request.ExpectResponse)
{
AddToCorrelationMatching(asyncItem);
}
using (var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _disposeToken.Token)) {
var timer = new Stopwatch();
try {
await _transport.ConnectAsync(cancellation.Token).ConfigureAwait(false);
var item = asyncItem;
_log.Info(() => LogEvent.Create($"Sending {request.ShortString()} (id {context.CorrelationId}, {item.RequestBytes.Count} bytes) to {Endpoint}"));
_log.Debug(() => LogEvent.Create($"{request.ApiKey} -----> {Endpoint} {{Context:{context},\nRequest:{request}}}"));
_configuration.OnWriting?.Invoke(Endpoint, request.ApiKey);
timer.Start();
var bytesWritten = await _transport.WriteBytesAsync(asyncItem.RequestBytes, cancellation.Token, context.CorrelationId).ConfigureAwait(false);
timer.Stop();
_configuration.OnWritten?.Invoke(Endpoint, request.ApiKey, bytesWritten, timer.Elapsed);
if (!request.ExpectResponse)
{
return(default(T));
}
} catch (Exception ex) {
timer.Stop();
_configuration.OnWriteFailed?.Invoke(Endpoint, request.ApiKey, timer.Elapsed, ex);
RemoveFromCorrelationMatching(asyncItem, ex);
throw;
}
receivedBytes = await asyncItem.ReceiveTask.Task.ThrowIfCancellationRequested(cancellation.Token).ConfigureAwait(false);
}
}
var response = request.ToResponse(context, receivedBytes);
_log.Debug(() => LogEvent.Create($"{Endpoint} -----> {{Context:{context},\n{request.ApiKey}Response:{response}}}"));
return(response);
}
/// <summary>
/// The bulk processor.
/// </summary>
/// <param name="item"><see cref="TInput"/> to process</param>
/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
/// <returns><see cref="Task"/></returns>
protected override async Task Process(AsyncItem <TInput, TOutput> item, CancellationToken cancellationToken)
{
var policy = Policy
.Handle <Exception>(ex => !(ex is TaskCanceledException || ex is OperationCanceledException))
.WaitAndRetryAsync(_clusterOptions.RetryAttempt,
retryAttempt =>
TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, sleepDuration, retry, context) =>
{
if (retry >= _clusterOptions.RetryAttempt)
{
_logger.LogError(
$"Could not process item after {retry} retry times: {exception.Message}");
item.TaskCompletionSource.TrySetException(exception);
}
});
var policyResult = await policy.ExecuteAndCaptureAsync(async ct =>
{
try
{
if (CpuUsage > _clusterOptions.LimitCpuUsage)
{
var suspensionTime = (CpuUsage - _clusterOptions.LimitCpuUsage) / CpuUsage * 100;
await Task.Delay((int)suspensionTime, ct);
}
var result = await _remoteContract.ProcessRemotely(item.Item, NodeMetrics);
item.TaskCompletionSource.TrySetResult(result);
}
catch (Exception ex) when(ex is TaskCanceledException || ex is OperationCanceledException)
{
_logger.LogTrace("The item process has been cancelled.");
item.TaskCompletionSource.TrySetCanceled();
}
finally
{
Interlocked.Increment(ref _processedItems);
}
}, cancellationToken).ConfigureAwait(false);
if (policyResult.Outcome == OutcomeType.Failure)
{
item.TaskCompletionSource.TrySetException(policyResult.FinalException);
_logger.LogCritical(
policyResult.FinalException != null
? $"Could not process item: {policyResult.FinalException.Message}."
: "An error has occured while processing the item.");
}
}
static int set_OnCreateItem(IntPtr L)
{
object o = LuaScriptMgr.GetLuaObject(L, 1);
AsyncItem obj = (AsyncItem)o;
if (obj == null)
{
LuaTypes types = LuaDLL.lua_type(L, 1);
if (types == LuaTypes.LUA_TTABLE)
{
LuaDLL.luaL_error(L, "unknown member name OnCreateItem");
}
else
{
LuaDLL.luaL_error(L, "attempt to index OnCreateItem on a nil value");
}
}
LuaTypes funcType = LuaDLL.lua_type(L, 3);
if (funcType != LuaTypes.LUA_TFUNCTION)
{
obj.OnCreateItem = (Func <int, Transform, GameObject>)LuaScriptMgr.GetNetObject(L, 3, typeof(Func <int, Transform, GameObject>));
}
else
{
LuaFunction func = LuaScriptMgr.ToLuaFunction(L, 3);
obj.OnCreateItem = (param0, param1) =>
{
int top = func.BeginPCall();
LuaScriptMgr.Push(L, param0);
LuaScriptMgr.Push(L, param1);
func.PCall(top, 2);
object[] objs = func.PopValues(top);
func.EndPCall(top);
return((GameObject)objs[0]);
};
}
return(0);
}
private void AsyncLogFlush(AsyncItem asyncItem, bool retry)
{
try
{
using (var fs = new FileStream(asyncItem.filepath, FileMode.Append, FileAccess.Write))
{
fs.Write(asyncItem.buffer, 0, asyncItem.buffer.Length);
}
}
catch (DirectoryNotFoundException)
{
//已为重试
if (retry)
{
//backgroup save ignore error
return;
}
//igore runing of remove rootpath
try
{
var dir = System.IO.Path.GetDirectoryName(asyncItem.filepath);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
}
catch (Exception exception)
{
this.TriggerError(asyncItem, exception);
}
this.AsyncLogFlush(asyncItem, true);
}
catch (Exception exception)
{
//backgroup save ignore error
this.TriggerError(asyncItem, exception);
}
}
static int get_OnCreateItem(IntPtr L)
{
object o = LuaScriptMgr.GetLuaObject(L, 1);
AsyncItem obj = (AsyncItem)o;
if (obj == null)
{
LuaTypes types = LuaDLL.lua_type(L, 1);
if (types == LuaTypes.LUA_TTABLE)
{
LuaDLL.luaL_error(L, "unknown member name OnCreateItem");
}
else
{
LuaDLL.luaL_error(L, "attempt to index OnCreateItem on a nil value");
}
}
LuaScriptMgr.Push(L, obj.OnCreateItem);
return(1);
}
public void GetNextThrowsIfCalledBeforePreviousGetNextCompletes()
{
TaskCompletionSource <int>[] taskSources = Enumerable.Range(0, 2).Select(_ => new TaskCompletionSource <int>()).ToArray();
IAsyncEnumerator <int> enumerator = TaskDependentEnumerator();
AsyncItem <int> firstItemTask = enumerator.GetNextAsync();
Func <Task <Optional <int> > > despicableAct = async() => await enumerator.GetNextAsync();
despicableAct.ShouldThrow <PreviousItemNotCompletedException>();
async IAsyncEnumerator <int> TaskDependentEnumerator()
{
foreach (TaskCompletionSource <int> tcs in taskSources)
{
int item = await tcs.Task.ConfigureAwait(false);
await AsyncYield.Item(item);
}
return(await AsyncYield.Break <int>());
}
}
private void AsyncLogFlush(AsyncItem asyncItem)
{
this.AsyncLogFlush(asyncItem, false);
}
private async Task DedicatedReceiveAsync()
{
// only allow one reader to execute, dump out all other requests
if (Interlocked.Increment(ref _receiveCount) > 1)
{
return;
}
try {
var header = new byte[Request.IntegerByteSize + Request.CorrelationSize];
AsyncItem asyncItem = null;
while (!_disposeToken.IsCancellationRequested)
{
await Retry // use backoff so we don't take over the CPU when there's a failure
.WithBackoff(int.MaxValue, minimumDelay : TimeSpan.FromMilliseconds(5), maximumDelay : TimeSpan.FromSeconds(5))
.TryAsync(
async(retryAttempt, elapsed) => {
await _transport.ConnectAsync(_disposeToken.Token).ConfigureAwait(false);
// reading the header
if (asyncItem == null)
{
await _transport.ReadBytesAsync(new ArraySegment <byte>(header), _disposeToken.Token).ConfigureAwait(false);
var responseSize = BitConverter.ToInt32(header, 0).ToBigEndian();
var correlationId = BitConverter.ToInt32(header, Request.IntegerByteSize).ToBigEndian();
asyncItem = LookupByCorrelateId(correlationId, responseSize);
if (asyncItem.ResponseBytes.Count > 0)
{
_log.Error(LogEvent.Create($"Request id {correlationId} matched a previous response ({asyncItem.ResponseBytes.Count + Request.CorrelationSize} of {responseSize + Request.CorrelationSize} bytes), now overwriting with {responseSize}? bytes"));
}
asyncItem.ResponseBytes = new ArraySegment <byte>(new byte[responseSize - Request.CorrelationSize]);
}
// reading the body
await _transport.ReadBytesAsync(asyncItem.ResponseBytes, CancellationToken.None).ConfigureAwait(false);
asyncItem.ResponseCompleted(_log);
asyncItem = null;
return(new RetryAttempt <bool>(true));
},
(exception, retryAttempt, retryDelay) => {
if (_disposeToken.IsCancellationRequested)
{
exception.PrepareForRethrow();
_disposeToken.Token.ThrowIfCancellationRequested(); // in case exception is null
}
if (exception is ConnectionException)
{
_log.Warn(() => LogEvent.Create(exception, "Socket has been re-established, so recovery of the remaining of the current message is uncertain at best"));
try {
asyncItem.ResponseCompleted(_log);
} catch {
// ignore
}
asyncItem = null;
}
_log.Write(retryAttempt == 0 ? LogLevel.Error : LogLevel.Info, () => LogEvent.Create(exception, $"Failed receiving from {Endpoint} on retry {retryAttempt}: Will retry in {retryDelay}"));
},
_disposeToken.Token).ConfigureAwait(false);
}
} catch (Exception ex) {
_log.Debug(() => LogEvent.Create(ex));
} finally {
await DisposeAsync().ConfigureAwait(false);
Interlocked.Decrement(ref _receiveCount);
_log.Info(() => LogEvent.Create($"Stopped receiving from {Endpoint}"));
}
}
public void AddAsyncOperation(AsyncItem async)
{
m_AsyncList.AddLast(async);
}
