private IScope BeforeWrappedMethod(string name, ILambdaContext lambdaContext, ISpanContext distributedTraceContext, object input = null)
{
// Should we use _tracer.ScopeManager.Active instead? It returns null if none is active;
IScope scope = null;
try
{
var tags = IOParser.ParseRequest(input);
var extractAdapter = new TextMapExtractAdapter(tags);
var spanContext = distributedTraceContext ?? _tracer.Extract(BuiltinFormats.TextMap, extractAdapter);
tags.Remove("newrelic");
scope = _tracer
.BuildSpan(name)
.AsChildOf(spanContext)
.WithTag("aws.requestId", lambdaContext.AwsRequestId ?? string.Empty)
.WithTag("aws.arn", lambdaContext.InvokedFunctionArn ?? string.Empty)
.StartActive();
DetectColdStart(scope, ref _isColdStart);
if (input != null)
{
AddTagsToActiveSpan(scope.Span, "request", tags);
}
}
catch (Exception exception)
{
Logger.Log(message: exception.ToString(), level: "ERROR");
}
return(scope);
}
public void TestInjectExtract()
{
var tracer = new WavefrontTracer
.Builder(new ConsoleReporter(DefaultSource), BuildApplicationTags())
.Build();
var span = tracer.BuildSpan("testOp").Start();
Assert.NotNull(span);
span.SetBaggageItem("customer", "testCustomer");
span.SetBaggageItem("requestType", "mobile");
var dictionary = new Dictionary <string, string>();
var textMapInjectAdapter = new TextMapInjectAdapter(dictionary);
tracer.Inject(span.Context, BuiltinFormats.TextMap, textMapInjectAdapter);
var textMapExtractAdapter = new TextMapExtractAdapter(dictionary);
var context =
(WavefrontSpanContext)tracer.Extract(BuiltinFormats.TextMap, textMapExtractAdapter);
Assert.Equal("testCustomer", context.GetBaggageItem("customer"));
Assert.Equal("mobile", context.GetBaggageItem("requesttype"));
}
/// <summary>
/// 开始执行Action
/// </summary>
/// <param name="context"></param>
public void OnActionExecuting(ActionExecutingContext context)
{
var path = context.HttpContext.Request.Path;
var callingHeaders = new TextMapExtractAdapter(context.HttpContext.Request.Headers.ToDictionary(m => m.Key, m => m.Value.ToString()));
var spanContex = _tracer.Extract(BuiltinFormats.HttpHeaders, callingHeaders);
ISpanBuilder builder = null;
if (spanContex != null)
{
builder = _tracer.BuildSpan("中间件Span").AsChildOf(spanContex);
}
else
{
builder = _tracer.BuildSpan("中间件Span");
}
_scope = builder.StartActive(true);
_scope.Span.SetOperationName(path);
// 记录请求信息到span
foreach (var query in context.HttpContext.Request.Query)
{
_scope.Span.SetTag(query.Key, query.Value);
}
if (context.HttpContext.Request.HasFormContentType)
{
foreach (var form in context.HttpContext.Request.Form)
{
_scope.Span.SetTag(form.Key, form.Value);
}
}
}
private async Task OnMessage <TMessage>(IMessage <TMessage> message, Func <TMessage, Task> processMessage)
{
var stringDictionary = message.Properties.Headers.ToDictionary(x => x.Key, x => Encoding.UTF8.GetString((byte[])x.Value));
var textMapExtractAdapter = new TextMapExtractAdapter(stringDictionary);
var spanContext = _tracer.Extract(BuiltinFormats.HttpHeaders, textMapExtractAdapter);
using (_tracer.BuildSpan(nameof(OnMessage)).AddReference(References.FollowsFrom, spanContext).StartActive(true))
using (_logger.BeginScope(new[] { new KeyValuePair <string, object>(nameof(_tracer.ActiveSpan.Context.TraceId), _tracer.ActiveSpan.Context.TraceId) }))
{
var tryCount = 0;
//TODO надо фильтровать ошибки базы, с3 и реббита, но тогда цикл должен быть выше по абстракции
while (true)
{
try
{
await processMessage(message.Body).ConfigureAwait(false);
break;
}
catch (Exception e)
{
tryCount++;
_logger.LogError(e, $"Process error, try count {tryCount}");
if (tryCount >= 5)
{
throw;
}
}
}
}
}
/// <summary>
/// Обработка вернувшихся сообщений
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void OnMessageReturned(object sender, EasyNetQ.MessageReturnedEventArgs args)
{
var stringDictionary = args.MessageProperties.Headers.ToDictionary(x => x.Key, x => Encoding.UTF8.GetString((byte[])x.Value));
var textMapExtractAdapter = new TextMapExtractAdapter(stringDictionary);
var spanContext = _tracer.Extract(BuiltinFormats.HttpHeaders, textMapExtractAdapter);
using (_tracer.BuildSpan(nameof(OnMessageReturned)).AddReference(References.ChildOf, spanContext).StartActive(true))
using (_logger.BeginScope(new[] { new KeyValuePair <string, object>(nameof(_tracer.ActiveSpan.Context.TraceId), _tracer.ActiveSpan.Context.TraceId) }))
{
_tracer.ActiveSpan.SetTag(Tags.Error, true);
if (args.MessageProperties.Headers.TryGetValue(MessageKeyHeader, out var bytes) && bytes is byte[] value)
{
var key = Encoding.UTF8.GetString(value);
_logger.LogError("Message returned {Exchange} {RoutingKey} reason {ReturnReason} {Key}",
args.MessageReturnedInfo.Exchange,
args.MessageReturnedInfo.RoutingKey,
args.MessageReturnedInfo.ReturnReason,
key);
MessageReturned?.Invoke(new MessageReturnedEventArgs(key));
}
else
{
_logger.LogError("Can not get error message filename");
}
}
}
/// <summary>
/// 调用管道
/// </summary>
/// <param name="context">上下文</param>
/// <param name="tracer">跟踪器</param>
/// <returns></returns>
public async Task InvokeAsync(HttpContext context, ITracer tracer)
{
_logger.LogInformation("jaeger调用");
var path = context.Request.Path;
if (Path.HasExtension(path))
{
await _next(context);
}
else
{
//接收传入的Headers
var callingHeaders = new TextMapExtractAdapter(context.Request.Headers.ToDictionary(m => m.Key, m => m.Value.ToString()));
var spanContex = tracer.Extract(BuiltinFormats.HttpHeaders, callingHeaders);
ISpanBuilder builder = null;
if (spanContex != null)
{
builder = tracer.BuildSpan("中间件Span").AsChildOf(spanContex);
}
else
{
builder = tracer.BuildSpan("中间件Span");
}
//开始设置Span
using (IScope scope = builder.StartActive(true))
{
scope.Span.SetOperationName(path);
// 记录请求信息到span
if (_jaegerOptions.IsQuerySpan)
{
foreach (var query in context.Request.Query)
{
//包含敏感词跳出
if (_jaegerOptions.NoSpanKeys.Contains(query.Key))
{
continue;
}
var value = query.Value.ToString().Length > _jaegerOptions.QueryValueMaxLength ? query.Value.ToString()?.Substring(0, _jaegerOptions.QueryValueMaxLength) : query.Value.ToString();
scope.Span.SetTag(query.Key, value);
}
}
if (_jaegerOptions.IsFormSpan && context.Request.HasFormContentType)
{
foreach (var form in context.Request.Form)
{
//包含敏感词跳出
if (_jaegerOptions.NoSpanKeys.Contains(form.Key))
{
continue;
}
var value = form.Value.ToString().Length > _jaegerOptions.FormValueMaxLength ? form.Value.ToString()?.Substring(0, _jaegerOptions.FormValueMaxLength) : form.Value.ToString();
scope.Span.SetTag(form.Key, value);
}
}
await _next(context);
}
}
}
public Tracer(string operaName, string spanContextStr)
{
Dictionary <string, string> dic = new Dictionary <string, string>();
dic.Add("uber-trace-id", spanContextStr);
var callingHeaders = new TextMapExtractAdapter(dic);
var extractedContext = GlobalTracer.Instance.Extract(BuiltinFormats.HttpHeaders, callingHeaders);
Scope = GlobalTracer.Instance.BuildSpan(operaName).AsChildOf(extractedContext).StartActive();
}
public void TestInvalidTraceIdExtract()
{
string val = ":7499dd16d98ab60e:3771de7e09c55ae8:1";
var dictionary = new Dictionary <string, string>();
dictionary.Add(JaegerHeader, val);
var textMapExtractAdapter = new TextMapExtractAdapter(dictionary);
WavefrontSpanContext ctx = wfJaegerPropagator.Extract(textMapExtractAdapter);
Assert.Null(ctx);
}
protected ISpanBuilder CreateTracingSpanBuilder(string spanName, ITracer tracer, HttpRequest request)
{
var callingHeaders = new TextMapExtractAdapter(request.Headers.ToDictionary(m => m.Key, m => m.Value.ToString()));
var spanContex = tracer.Extract(BuiltinFormats.HttpHeaders, callingHeaders);
if (spanContex != null)
{
return(tracer.BuildSpan(spanName).AsChildOf(spanContex));
}
else
{
return(tracer.BuildSpan(spanName));
}
}
private static ISpanContext ExtractSpanContextOrDefault(
ITracer tracer,
ServerCallContext context,
ISpanContext defaultVal = null)
{
ISpanContext spanContext = defaultVal;
try {
var dict = MetadataToDictionary(context.RequestHeaders);
var adapter = new TextMapExtractAdapter(dict);
spanContext = tracer.Extract(BuiltinFormats.HttpHeaders, adapter);
} catch {}
return(spanContext);
}
public void TestTraceIdExtractEncoded()
{
string val = "3871de7e09c53ae8%3A7499dd16d98ab60e%3A3771de7e09c55ae8%3A1";
var dictionary = new Dictionary <string, string>();
dictionary.Add(JaegerHeader, val);
var textMapExtractAdapter = new TextMapExtractAdapter(dictionary);
WavefrontSpanContext ctx = wfJaegerPropagator.Extract(textMapExtractAdapter);
Assert.NotNull(ctx);
Assert.Equal("00000000-0000-0000-3871-de7e09c53ae8", ctx.TraceId);
Assert.Equal("00000000-0000-0000-7499-dd16d98ab60e", ctx.SpanId);
Assert.Equal("00000000-0000-0000-7499-dd16d98ab60e", ctx.GetBaggageItem(ParentIdKey));
Assert.True(ctx.GetSamplingDecision() ?? false);
}
public async Task InvokeAsync(HttpContext context, ITracer tracer)
{
var path = context.Request.Path;
if (Path.HasExtension(path))
{
await _next(context);
}
else
{
var callingHeaders = new TextMapExtractAdapter(context.Request.Headers.ToDictionary(m => m.Key, m => m.Value.ToString()));
var spanContex = tracer.Extract(BuiltinFormats.HttpHeaders, callingHeaders);
ISpanBuilder builder = null;
if (spanContex != null)
{
builder = tracer.BuildSpan("中间件Span").AsChildOf(spanContex);
}
else
{
builder = tracer.BuildSpan("中间件Span");
}
using (IScope scope = builder.StartActive(true))
{
scope.Span.SetOperationName(path);
// 记录请求信息到span
foreach (var query in context.Request.Query)
{
scope.Span.SetTag(query.Key, query.Value);
}
if (context.Request.HasFormContentType)
{
foreach (var form in context.Request.Form)
{
scope.Span.SetTag(form.Key, form.Value);
}
}
await _next(context);
}
}
}
public void TestDebugCorrelationId()
{
Tracer tracer = new Tracer.Builder("test")
.WithReporter(new InMemoryReporter())
.WithSampler(new ConstSampler(true))
.Build();
var headers = new Dictionary <string, string>();
headers.Add(Constants.DebugIdHeaderKey, "Coraline");
ITextMap carrier = new TextMapExtractAdapter(headers);
SpanContext spanContext = (SpanContext)tracer.Extract(BuiltinFormats.TextMap, carrier);
Assert.True(spanContext.IsDebugIdContainerOnly());
Assert.Equal("Coraline", spanContext.DebugId);
Span span = (Span)tracer.BuildSpan("span").AsChildOf(spanContext).Start();
spanContext = (SpanContext)span.Context;
Assert.True(spanContext.IsSampled);
Assert.True(spanContext.IsDebug);
Assert.Equal("Coraline", span.GetTags()[Constants.DebugIdHeaderKey]);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
//Begin code instrumentation - reference https://github.com/wavefrontHQ/wavefront-opentracing-sdk-csharp
//The application, service, cluster, and shard variables are all metadata to be added to each span created.
string application = "VMworld2020Demo";
string service = "GlobalDataAggregator";
string cluster = "Azure";
string shard = "networknerd4";
//The URL and token are for direct ingestion of metrics, traces, and spans (no proxy in use here).
//The API token can be found inside the Tanzu Observability (Wavefront) web UI and is unique to your environment. Click the gear icon in the upper right, click your e-mail address, and then select API Access.
string wfURL = "https://vmware.wavefront.com";
string token = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx";
// Create ApplicationTags - for tracing purposes
ApplicationTags applicationTags = new ApplicationTags.Builder(application, service).Cluster(cluster).Shard(shard).Build();
//Configure a MetricsBuilder object - for custom metrics sent via the Metrics SDK
var MyMetricsBuilder = new MetricsBuilder();
//Initialize WavefrontDirectIngestionClient
WavefrontDirectIngestionClient.Builder wfDirectIngestionClientBuilder = new WavefrontDirectIngestionClient.Builder(wfURL, token);
// Create an IWavefrontSender instance for sending data via direct ingestion.
IWavefrontSender wavefrontSender = wfDirectIngestionClientBuilder.Build();
//Configure MeetricsBuilder to Report to Wavefront with proper sender object and source tag specified. In this case my source is the function name.
MyMetricsBuilder.Report.ToWavefront(
options =>
{
options.WavefrontSender = wavefrontSender;
options.Source = "TruckGlobalDataAggregator";
});
//Build IMetrics instance
var MyMetrics = MyMetricsBuilder.Build();
//These are arrays for key value pairs to add as metric tags. You can add some or many here as you instrument your code.
string[] keys = new string[3] {
"FunctionApp", "Cloud", "Region"
};
string[] values = new string[3] {
"networknerd4", "Azure", "Central-US"
};
// Configure and instantiate a DeltaCounter using DeltaCounterOptions.Builder. The metric name is azure.function.execution.deltacounter.
var myDeltaCounter = new DeltaCounterOptions.Builder("azure.function.execution.deltacounter").MeasurementUnit(Unit.Calls).Tags(new MetricTags(keys, values)).Build();
// Increment the counter by 1
MyMetrics.Measure.Counter.Increment(myDeltaCounter);
//Force reporting all custom metrics
await Task.WhenAll(MyMetrics.ReportRunner.RunAllAsync());
//Create a WavefrontSpanReporter for reporting trace data that originates on <sourceName>. The source is the function name in this case.
IReporter wfSpanReporter = new WavefrontSpanReporter.Builder()
.WithSource("TruckGlobalDataAggregator").Build(wavefrontSender);
//Create CompositeReporter and ConsoleReporter objects for more OpenTracing metrics
IReporter consoleReporter = new ConsoleReporter("TruckGlobalDataAggregator");
IReporter compositeReporter = new CompositeReporter(wfSpanReporter, consoleReporter);
//Create the WavefrontTracer.
WavefrontTracer MyTracer = new WavefrontTracer.Builder(wfSpanReporter, applicationTags).Build();
//The variable MyDictionary is needed to extract span context in case a call is made from another function / outside this function.
IDictionary <string, string> MyDictionary = new Dictionary <string, string>();
foreach (var entry in req.Headers)
{
MyDictionary.TryAdd(entry.Key, entry.Value);
}
//Attempt to pull span fontext from HTTP headers passed into this function to continue a span across environments. The proper context will be loaded into the variable
//ctx if so. The second line of code loads all metadata from the span context.
ITextMap carrier = new TextMapExtractAdapter(MyDictionary);
OpenTracing.ISpanContext ctx = MyTracer.Extract(BuiltinFormats.HttpHeaders, carrier);
OpenTracing.IScope receivingScope = MyTracer.BuildSpan("TruckGlobalDataAggregator.Execute").AsChildOf(ctx).StartActive(true);
//Start building a new span called TruckGlobalDataAggregator.Execute if there was no context passed into headers.
if (MyTracer.ActiveSpan != null)
{
MyTracer.BuildSpan("TruckGlobalDataAggregator.Execute").StartActive();
}
log.LogInformation("C# HTTP trigger function processed a request.");
string name = req.Query["name"];
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
name = name ?? data?.name;
//Add function execution delays based on input - for personal testing only.
if (string.Equals(name, "0.5"))
{
await Task.Delay(500);
}
if (string.Equals(name, "1"))
{
await Task.Delay(1000);
}
if (string.Equals(name, "1.5"))
{
await Task.Delay(1500);
}
if (string.Equals(name, "2"))
{
await Task.Delay(2000);
}
string responseMessage = string.IsNullOrEmpty(name)
? "This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response."
: $"Hello, {name}. This HTTP triggered function executed successfully.";
//Finish the span
MyTracer.ActiveSpan.Finish();
//Close the tracer before application exit
MyTracer.Close();
return(new OkObjectResult(responseMessage));
}
