/// <summary>调用</summary>
/// <param name="ctx"></param>
/// <returns></returns>
public async Task Invoke(HttpContext ctx)
{
// APM跟踪
//var span = Tracer?.NewSpan(ctx.Request.Path);
ISpan span = null;
if (Tracer != null)
{
var action = GetAction(ctx);
if (!action.IsNullOrEmpty())
{
span = Tracer.NewSpan(action);
span.Tag = ctx.Request.GetRawUrl() + "";
span.Detach(ctx.Request.Headers.ToDictionary(e => e.Key, e => (Object)e.Value));
}
}
try
{
await _next.Invoke(ctx);
}
catch (Exception ex)
{
span?.SetError(ex, null);
throw;
}
finally
{
span?.Dispose();
}
}
public async Task DisjointThreads_Annotate()
{
var mockConsumer = new Mock <IConsumer <TraceProto> >();
var tracer = SimpleManagedTracer.Create(mockConsumer.Object, ProjectId, TraceId);
mockConsumer.Setup(c => c.Receive(
Match.Create <IEnumerable <TraceProto> >(
tProto => IsValidSpan(tProto.Single().Spans.Single(), "span-name"))));
var annotation = new Dictionary <string, string> {
{ "new", "label" }
};
Predicate <IEnumerable <TraceProto> > matcher = t =>
{
var singleSpan = t.Single().Spans.Single();
return(IsValidSpan(singleSpan, "span-name") &&
string.IsNullOrWhiteSpace(singleSpan.Labels[TraceLabels.StackTrace]) &&
TraceUtils.IsValidAnnotation(singleSpan, annotation));
};
mockConsumer.Setup(c => c.Receive(Match.Create(matcher)));
ISpan span = null;
await RunInDisjointThreads(() => span = tracer.StartSpan("span-name"),
() =>
{
span.SetStackTrace(FilledStackTrace);
span.AnnotateSpan(annotation);
span.Dispose();
});
mockConsumer.Verify();
}
/// <summary>调用</summary>
/// <param name="ctx"></param>
/// <returns></returns>
public async Task Invoke(HttpContext ctx)
{
// APM追踪
//var span = Tracer?.NewSpan(ctx.Request.Path);
ISpan span = null;
if (Tracer != null)
{
var action = GetAction(ctx);
if (!action.IsNullOrEmpty())
{
span = Tracer.NewSpan(action);
span.Detach(ctx.Request.Headers.ToDictionary(e => e.Key, e => (Object)e.Value));
}
}
ManageProvider.UserHost = ctx.GetUserHost();
try
{
await _next.Invoke(ctx);
}
catch (Exception ex)
{
span?.SetError(ex, ctx.Request.QueryString + "");
throw;
}
finally
{
ManageProvider.UserHost = null;
span?.Dispose();
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="onException">异常处理</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="tracer">性能跟踪</param>
/// <returns></returns>
public static async Task RunLoopAsync <T>(this IProducerConsumer <String> queue, Func <T, Task> onMessage, Action <Exception> onException = null, CancellationToken cancellationToken = default, ITracer tracer = null)
{
// 主题
var topic = (queue as RedisBase).Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
// 超时时间，用于阻塞等待
var timeout = 2;
if (queue is RedisBase rb && rb.Redis != null)
{
timeout = rb.Redis.Timeout / 1000 - 1;
}
while (!cancellationToken.IsCancellationRequested)
{
ISpan span = null;
try
{
// 异步阻塞消费
var msg = await queue.TakeOneAsync(timeout);
if (msg != null)
{
// 反序列化消息
var message = msg.ToJsonEntity <T>();
span = tracer?.NewSpan($"mq:{topic}", msg);
// 处理消息
await onMessage(message);
// 确认消息
queue.Acknowledge(msg);
}
else
{
// 没有消息，歇一会
await TaskEx.Delay(1000);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
onException?.Invoke(ex);
}
finally
{
span?.Dispose();
}
}
}
public void TestTracerId()
{
var tracer = new DefaultTracer();
// 内嵌片段，应该共用TraceId
{
using var span = tracer.NewSpan("test");
Thread.Sleep(100);
{
Assert.Equal(span, DefaultSpan.Current);
using var span2 = tracer.NewSpan("test2");
Assert.Equal(span2, DefaultSpan.Current);
Assert.Equal(span.TraceId, span2.TraceId);
Assert.Equal(span.Id, span2.ParentId);
Assert.NotEqual(span.Id, span2.Id);
Thread.Sleep(100);
{
using var span3 = tracer.NewSpan("test3");
Assert.Equal(span3, DefaultSpan.Current);
Assert.Equal(span.TraceId, span3.TraceId);
Assert.Equal(span2.Id, span3.ParentId);
Assert.NotEqual(span2.Id, span3.Id);
}
Assert.Equal(span2, DefaultSpan.Current);
}
Assert.Equal(span, DefaultSpan.Current);
}
// 内嵌片段，不同线程应该使用不同TraceId
{
using var span = tracer.NewSpan("test");
Thread.Sleep(100);
// 另一个线程建立span，必须用UnsafeQueueUserWorkItem截断上下文传递，否则还是会建立父子关系
ISpan span2 = null;
ThreadPool.UnsafeQueueUserWorkItem(s =>
{
span2 = tracer.NewSpan("test2");
}, null);
Thread.Sleep(100);
//using var span2 = Task.Factory.StartNew(() => tracer.NewSpan("test2"), TaskCreationOptions.LongRunning).Result;
Assert.NotEqual(span.TraceId, span2.TraceId);
Assert.NotEqual(span.Id, span2.ParentId);
span2.Dispose();
}
var builder = tracer.BuildSpan("test");
Assert.Equal(2, builder.Total);
Assert.Equal(0, builder.Errors);
}
public void TestParentId()
{
var tracer = new DefaultTracer();
// QueueUserWorkItem传递上下文
{
using var span = tracer.NewSpan("test");
Thread.Sleep(100);
// 另一个线程建立span
ISpan span2 = null;
var e = new AutoResetEvent(false);
ThreadPool.QueueUserWorkItem(s =>
{
span2 = tracer.NewSpan("test2");
e.Set();
}, null);
//Thread.Sleep(100);
e.WaitOne();
Assert.Equal(span.TraceId, span2.TraceId);
Assert.Equal(span.Id, span2.ParentId);
span2.Dispose();
}
// Task传递上下文
{
using var span = tracer.NewSpan("test");
Thread.Sleep(100);
// 另一个线程建立span
using var span2 = Task.Run(() => tracer.NewSpan("test2")).Result;
Assert.Equal(span.TraceId, span2.TraceId);
Assert.Equal(span.Id, span2.ParentId);
}
// Task传递上下文
{
using var span = tracer.NewSpan("test");
Thread.Sleep(100);
// 另一个线程建立span
using var span2 = Task.Factory.StartNew(() => tracer.NewSpan("test2"), TaskCreationOptions.LongRunning).Result;
Assert.Equal(span.TraceId, span2.TraceId);
Assert.Equal(span.Id, span2.ParentId);
}
var builder = tracer.BuildSpan("test");
Assert.Equal(3, builder.Total);
Assert.Equal(0, builder.Errors);
builder = tracer.BuildSpan("test2");
Assert.Equal(3, builder.Total);
Assert.Equal(0, builder.Errors);
}
public void Dispose()
{
_innerSpan.Dispose();
if (_active)
{
_traceContext.TryPop();
_active = false;
}
}
internal void EndRequest(object sender, EventArgs e)
{
ISpan span = ContextInstanceManager.Get <ISpan>();
if (span == null)
{
return;
}
// End the span and annotate it with information from the current response.
span.AnnotateSpan(Labels.FromHttpResponse(HttpContext.Current.Response));
span.Dispose();
}
public async Task DisjointThreads_Annotate_Fail()
{
var mockConsumer = new Mock <IConsumer <TraceProto> >();
var tracer = SimpleManagedTracer.Create(mockConsumer.Object, ProjectId, TraceId);
ISpan span = null;
await RunInDisjointThreads(() => span = tracer.StartSpan("span-name"), () =>
{
tracer.SetStackTrace(FilledStackTrace);
span.Dispose();
}, new InvalidOperationException());
mockConsumer.Verify(c => c.Receive(It.IsAny <IEnumerable <TraceProto> >()), Times.Never);
}
/// <summary>调用</summary>
/// <param name="ctx"></param>
/// <returns></returns>
public async Task Invoke(HttpContext ctx)
{
// APM跟踪
//var span = Tracer?.NewSpan(ctx.Request.Path);
ISpan span = null;
if (Tracer != null)
{
var action = GetAction(ctx);
if (!action.IsNullOrEmpty())
{
span = Tracer.NewSpan(action);
span.Tag = ctx.GetUserHost() + " " + ctx.Request.GetRawUrl();
span.Detach(ctx.Request.Headers.ToDictionary(e => e.Key, e => (Object)e.Value));
}
}
try
{
await _next.Invoke(ctx);
// 根据状态码识别异常
if (span != null)
{
var code = ctx.Response.StatusCode;
if (code >= 400)
{
span.SetError(new HttpRequestException($"Http Error {code} {(HttpStatusCode)code}"), null);
}
}
}
catch (Exception ex)
{
span?.SetError(ex, null);
throw;
}
finally
{
span?.Dispose();
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this RedisStream <String> queue, Func <T, Message, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null)
{
// 大循环之前，打断性能追踪调用链
DefaultSpan.Current = null;
// 自动创建消费组
var gis = queue.GetGroups();
if (gis == null || !queue.Group.IsNullOrEmpty() && !gis.Any(e => e.Name.EqualIgnoreCase(queue.Group)))
{
queue.GroupCreate(queue.Group);
}
// 主题
var topic = queue.Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
var rds = queue.Redis;
var tracer = rds.Tracer;
var errLog = log ?? XTrace.Log;
// 超时时间，用于阻塞等待
var timeout = rds.Timeout / 1000 - 1;
while (!cancellationToken.IsCancellationRequested)
{
Message mqMsg = null;
ISpan span = null;
try
{
// 异步阻塞消费
mqMsg = await queue.TakeMessageAsync(timeout, cancellationToken);
if (mqMsg != null)
{
// 埋点
span = tracer?.NewSpan($"redismq:{topic}", mqMsg);
log?.Info($"[{topic}]消息内容为：{mqMsg}");
var bodys = mqMsg.Body;
for (var i = 0; i < bodys.Length; i++)
{
if (bodys[i].EqualIgnoreCase("traceParent") && i + 1 < bodys.Length)
{
span.Detach(bodys[i + 1]);
}
}
// 解码
var msg = mqMsg.GetBody <T>();
// 处理消息
await onMessage(msg, mqMsg, cancellationToken);
// 确认消息
queue.Acknowledge(mqMsg.Id);
}
else
{
// 没有消息，歇一会
await Task.Delay(1000, cancellationToken);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
errLog?.Error("[{0}/{1}]消息处理异常：{2} {3}", topic, mqMsg?.Id, mqMsg?.ToJson(), ex);
}
finally
{
span?.Dispose();
}
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <param name="idField">消息标识字段名，用于处理错误重试</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this IProducerConsumer <String> queue, Func <T, String, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null)
{
// 大循环之前，打断性能追踪调用链
DefaultSpan.Current = null;
// 主题
var topic = (queue as RedisBase).Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
var rds = (queue as RedisBase).Redis;
var tracer = rds.Tracer;
var errLog = log ?? XTrace.Log;
var ids = new List <String> {
"Id", "guid", "OrderId", "Code"
};
if (!idField.IsNullOrEmpty() && !ids.Contains(idField))
{
ids.Insert(0, idField);
}
// 超时时间，用于阻塞等待
var timeout = rds.Timeout / 1000 - 1;
while (!cancellationToken.IsCancellationRequested)
{
var msgId = "";
var mqMsg = "";
ISpan span = null;
try
{
// 异步阻塞消费
mqMsg = await queue.TakeOneAsync(timeout);
if (mqMsg != null)
{
// 埋点
span = tracer?.NewSpan($"redismq:{topic}", mqMsg);
log?.Info($"[{topic}]消息内容为：{mqMsg}");
// 解码
var dic = JsonParser.Decode(mqMsg);
var msg = JsonHelper.Convert <T>(dic);
if (dic.TryGetValue("traceParent", out var tp))
{
span.Detach(tp + "");
}
// 消息标识
foreach (var item in ids)
{
if (dic.TryGetValue(item, out var id))
{
msgId = id + "";
if (!msgId.IsNullOrEmpty())
{
break;
}
}
}
// 处理消息
await onMessage(msg, mqMsg, cancellationToken);
// 确认消息
queue.Acknowledge(mqMsg);
}
else
{
// 没有消息，歇一会
await Task.Delay(1000, cancellationToken);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
// 消息处理错误超过10次则抛弃
if (!mqMsg.IsNullOrEmpty())
{
if (msgId.IsNullOrEmpty())
{
msgId = mqMsg.MD5();
}
errLog?.Error("[{0}/{1}]消息处理异常：{2} {3}", topic, msgId, mqMsg, ex);
var key = $"{topic}:Error:{msgId}";
var rs = rds.Increment(key, 1);
if (rs < 10)
{
rds.SetExpire(key, TimeSpan.FromHours(24));
}
else
{
queue.Acknowledge(mqMsg);
errLog?.Error("[{0}/{1}]错误过多，删除消息", topic, msgId);
}
}
}
finally
{
span?.Dispose();
}
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <param name="idField">消息标识字段名，用于处理错误重试</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this RedisReliableQueue <String> queue, Func <T, String, CancellationToken, Task> onMessage, CancellationToken cancellationToken = default, ILog log = null, String idField = null)
{
// 大循环之前，打断性能追踪调用链
DefaultSpan.Current = null;
// 主题
var topic = queue.Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
var rds = queue.Redis;
var tracer = rds.Tracer;
var errLog = log ?? XTrace.Log;
// 备用redis，容错、去重
var rds2 = new FullRedis
{
Name = rds.Name + "Bak",
Server = rds.Server,
UserName = rds.UserName,
Password = rds.Password,
Db = rds.Db == 15 ? 0 : (rds.Db + 1),
Tracer = rds.Tracer,
};
// 消息去重
if (queue.DuplicateExpire > 0 && idField.IsNullOrEmpty())
{
throw new ArgumentNullException(nameof(idField), $"队列[{topic}]消息[{queue.DuplicateExpire}]秒去重，需要指定消息唯一标识idField");
}
var ids = new List <String> {
"Id", "guid", "OrderId", "Code"
};
if (!idField.IsNullOrEmpty() && !ids.Contains(idField))
{
ids.Insert(0, idField);
}
// 超时时间，用于阻塞等待
var timeout = rds.Timeout / 1000 - 1;
while (!cancellationToken.IsCancellationRequested)
{
var msgId = "";
var mqMsg = "";
ISpan span = null;
try
{
// 异步阻塞消费
mqMsg = await queue.TakeOneAsync(timeout, cancellationToken);
if (mqMsg != null)
{
// 埋点
span = tracer?.NewSpan($"redismq:{topic}", mqMsg);
log?.Info($"[{topic}]消息内容为：{mqMsg}");
// 解码
var dic = JsonParser.Decode(mqMsg);
var msg = JsonHelper.Convert <T>(dic);
if (dic.TryGetValue("traceParent", out var tp))
{
span.Detach(tp + "");
}
// 消息标识
foreach (var item in ids)
{
if (dic.TryGetValue(item, out var id))
{
msgId = id + "";
if (!msgId.IsNullOrEmpty())
{
break;
}
}
}
// 消息去重
if (queue.DuplicateExpire > 0)
{
// 抢占msgId，处理异常时退出抢占
var dkey = $"{topic}:Duplicate:{msgId}";
if (!rds2.Add(dkey, queue.Status.Key, queue.DuplicateExpire))
{
log?.Info("队列[{0}]遇到重复消息[{1}]，自动跳过", topic, msgId);
}
else
{
try
{
await onMessage(msg, mqMsg, cancellationToken);
}
catch
{
rds2.Remove(dkey);
throw;
}
}
}
else
{
// 处理消息
await onMessage(msg, mqMsg, cancellationToken);
}
// 确认消息
queue.Acknowledge(mqMsg);
}
else
{
// 没有消息，歇一会
await Task.Delay(1000, cancellationToken);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
// 消息处理错误超过10次则抛弃
if (!mqMsg.IsNullOrEmpty())
{
if (msgId.IsNullOrEmpty())
{
msgId = mqMsg.MD5();
}
errLog?.Error("[{0}/{1}]消息处理异常：{2} {3}", topic, msgId, mqMsg, ex);
var key = $"{topic}:Error:{msgId}";
var rs = rds2.Increment(key, 1);
if (rs < 10)
{
rds2.SetExpire(key, TimeSpan.FromHours(24));
}
else
{
queue.Acknowledge(mqMsg);
errLog?.Error("[{0}/{1}]错误过多，删除消息", topic, msgId);
}
}
}
finally
{
span?.Dispose();
}
}
}
/// <summary>队列消费大循环，处理消息后自动确认</summary>
/// <typeparam name="T">消息类型</typeparam>
/// <param name="queue">队列</param>
/// <param name="onMessage">消息处理。如果处理消息时抛出异常，消息将延迟后回到队列</param>
/// <param name="cancellationToken">取消令牌</param>
/// <param name="log">日志对象</param>
/// <returns></returns>
public static async Task ConsumeAsync <T>(this RedisReliableQueue <String> queue, Action <String> onMessage, CancellationToken cancellationToken = default, ILog log = null)
{
// 大循环之前，打断性能追踪调用链
DefaultSpan.Current = null;
// 主题
var topic = queue.Key;
if (topic.IsNullOrEmpty())
{
topic = queue.GetType().Name;
}
var rds = queue.Redis;
var tracer = rds.Tracer;
var errLog = log ?? XTrace.Log;
// 备用redis，容错、去重
var rds2 = new FullRedis
{
Name = rds.Name + "Bak",
Server = rds.Server,
UserName = rds.UserName,
Password = rds.Password,
Db = rds.Db == 15 ? 0 : (rds.Db + 1),
Tracer = rds.Tracer,
};
// 超时时间，用于阻塞等待
var timeout = rds.Timeout / 1000 - 1;
while (!cancellationToken.IsCancellationRequested)
{
var mqMsg = "";
ISpan span = null;
try
{
// 异步阻塞消费
mqMsg = await queue.TakeOneAsync(timeout, cancellationToken);
if (mqMsg != null)
{
// 埋点
span = tracer?.NewSpan($"redismq:{topic}", mqMsg);
log?.Info($"[{topic}]消息内容为：{mqMsg}");
// 处理消息
onMessage(mqMsg);
// 确认消息
queue.Acknowledge(mqMsg);
}
else
{
// 没有消息，歇一会
await Task.Delay(1000, cancellationToken);
}
}
catch (ThreadAbortException) { break; }
catch (ThreadInterruptedException) { break; }
catch (Exception ex)
{
span?.SetError(ex, null);
// 消息处理错误超过10次则抛弃
if (!mqMsg.IsNullOrEmpty())
{
var msgId = mqMsg.MD5();
errLog?.Error("[{0}/{1}]消息处理异常：{2} {3}", topic, msgId, mqMsg, ex);
var key = $"{topic}:Error:{msgId}";
var rs = rds2.Increment(key, 1);
if (rs < 10)
{
rds2.SetExpire(key, TimeSpan.FromHours(24));
}
else
{
queue.Acknowledge(mqMsg);
errLog?.Error("[{0}/{1}]错误过多，删除消息", topic, msgId);
}
}
}
finally
{
span?.Dispose();
}
}
}
public async Task DisjointThreads()
{
var mockConsumer = new Mock <IConsumer <TraceProto> >();
var tracer = SimpleManagedTracer.Create(mockConsumer.Object, ProjectId, TraceId);
mockConsumer.Setup(c => c.Receive(
Match.Create <IEnumerable <TraceProto> >(
tProto => IsValidSpan(tProto.Single().Spans.Single(), "span-name"))));
ISpan span = null;
await RunInDisjointThreads(() => span = tracer.StartSpan("span-name"), () => span.Dispose());
mockConsumer.VerifyAll();
}
