protected async Task AssertHttpSpan(
string path,
MockTracerAgent agent,
int httpPort,
HttpStatusCode expectedHttpStatusCode,
string expectedSpanType,
string expectedOperationName,
string expectedResourceName)
{
IImmutableList <MockTracerAgent.Span> spans;
using (var httpClient = new HttpClient())
{
// disable tracing for this HttpClient request
httpClient.DefaultRequestHeaders.Add(HttpHeaderNames.TracingEnabled, "false");
var testStart = DateTime.UtcNow;
var response = await httpClient.GetAsync($"http://localhost:{httpPort}" + path);
var content = await response.Content.ReadAsStringAsync();
Output.WriteLine($"[http] {response.StatusCode} {content}");
Assert.Equal(expectedHttpStatusCode, response.StatusCode);
spans = agent.WaitForSpans(
count: 1,
minDateTime: testStart,
operationName: expectedOperationName);
Assert.True(spans.Count == 1, "expected one span");
}
MockTracerAgent.Span span = spans[0];
Assert.Equal(expectedSpanType, span.Type);
Assert.Equal(expectedOperationName, span.Name);
Assert.Equal(expectedResourceName, span.Resource);
}
public void TracingDisabled_DoesNotSubmitsTraces()
{
const string expectedOperationName = "http.request";
int agentPort = TcpPortProvider.GetOpenPort();
int httpPort = TcpPortProvider.GetOpenPort();
using (var agent = new MockTracerAgent(agentPort))
using (ProcessResult processResult = RunSampleAndWaitForExit(agent.Port, arguments: $"TracingDisabled Port={httpPort}"))
{
Assert.True(processResult.ExitCode >= 0, $"Process exited with code {processResult.ExitCode}");
var spans = agent.WaitForSpans(1, 3000, operationName: expectedOperationName);
Assert.Equal(0, spans.Count);
var traceId = StringUtil.GetHeader(processResult.StandardOutput, HttpHeaderNames.TraceId);
var parentSpanId = StringUtil.GetHeader(processResult.StandardOutput, HttpHeaderNames.ParentId);
var tracingEnabled = StringUtil.GetHeader(processResult.StandardOutput, HttpHeaderNames.TracingEnabled);
Assert.Null(traceId);
Assert.Null(parentSpanId);
Assert.Equal("false", tracingEnabled);
}
}
public void HttpClient()
{
int expectedSpanCount = EnvironmentHelper.IsCoreClr() ? 2 : 1;
const string expectedOperationName = "http.request";
const string expectedServiceName = "Samples.HttpMessageHandler-http-client";
int agentPort = TcpPortProvider.GetOpenPort();
int httpPort = TcpPortProvider.GetOpenPort();
Output.WriteLine($"Assigning port {agentPort} for the agentPort.");
Output.WriteLine($"Assigning port {httpPort} for the httpPort.");
using (var agent = new MockTracerAgent(agentPort))
using (ProcessResult processResult = RunSampleAndWaitForExit(agent.Port, arguments: $"HttpClient Port={httpPort}"))
{
Assert.True(processResult.ExitCode >= 0, $"Process exited with code {processResult.ExitCode}");
var spans = agent.WaitForSpans(expectedSpanCount, operationName: expectedOperationName);
Assert.True(spans.Count >= expectedSpanCount, $"Expected at least {expectedSpanCount} span, only received {spans.Count}" + System.Environment.NewLine + "IMPORTANT: Make sure Datadog.Trace.ClrProfiler.Managed.dll and its dependencies are in the GAC.");
foreach (var span in spans)
{
Assert.Equal(expectedOperationName, span.Name);
Assert.Equal(expectedServiceName, span.Service);
Assert.Equal(SpanTypes.Http, span.Type);
Assert.Equal(nameof(HttpMessageHandler), span.Tags[Tags.InstrumentationName]);
}
var firstSpan = spans.First();
var traceId = GetHeader(processResult.StandardOutput, HttpHeaderNames.TraceId);
var parentSpanId = GetHeader(processResult.StandardOutput, HttpHeaderNames.ParentId);
Assert.Equal(firstSpan.TraceId.ToString(CultureInfo.InvariantCulture), traceId);
Assert.Equal(firstSpan.SpanId.ToString(CultureInfo.InvariantCulture), parentSpanId);
}
}
public void SubmitsTraces(string packageVersion)
{
int agentPort = TcpPortProvider.GetOpenPort();
using (var agent = new MockTracerAgent(agentPort))
using (var processResult = RunSampleAndWaitForExit(agent.Port, packageVersion: packageVersion))
{
Assert.True(processResult.ExitCode >= 0, $"Process exited with code {processResult.ExitCode}");
var expected = new List <string>();
// commands with sync and async
for (var i = 0; i < 2; i++)
{
expected.AddRange(new List <string>
{
"Bulk",
"Create",
"Search",
"DeleteByQuery",
"CreateIndex",
"IndexExists",
"UpdateIndexSettings",
"BulkAlias",
"GetAlias",
"PutAlias",
// "AliasExists",
"DeleteAlias",
"DeleteAlias",
"CreateIndex",
// "SplitIndex",
"DeleteIndex",
"CloseIndex",
"OpenIndex",
"PutIndexTemplate",
"IndexTemplateExists",
"DeleteIndexTemplate",
"IndicesShardStores",
"IndicesStats",
"DeleteIndex",
"GetAlias",
"CatAliases",
"CatAllocation",
"CatCount",
"CatFielddata",
"CatHealth",
"CatHelp",
"CatIndices",
"CatMaster",
"CatNodeAttributes",
"CatNodes",
"CatPendingTasks",
"CatPlugins",
"CatRecovery",
"CatRepositories",
"CatSegments",
"CatShards",
// "CatSnapshots",
"CatTasks",
"CatTemplates",
"CatThreadPool",
// "PutJob",
// "ValidateJob",
// "GetInfluencers",
// "GetJobs",
// "GetJobStats",
// "GetModelSnapshots",
// "GetOverallBuckets",
// "FlushJob",
// "ForecastJob",
// "GetAnomalyRecords",
// "GetBuckets",
// "GetCategories",
// "CloseJob",
// "OpenJob",
// "DeleteJob",
"ClusterAllocationExplain",
"ClusterGetSettings",
"ClusterHealth",
"ClusterPendingTasks",
"ClusterPutSettings",
"ClusterReroute",
"ClusterState",
"ClusterStats",
"PutRole",
// "PutRoleMapping",
"GetRole",
// "GetRoleMapping",
// "DeleteRoleMapping",
"DeleteRole",
"PutUser",
"ChangePassword",
"GetUser",
// "DisableUser",
"DeleteUser",
});
}
var spans = agent.WaitForSpans(expected.Count)
.Where(s => s.Type == "elasticsearch")
.OrderBy(s => s.Start)
.ToList();
foreach (var span in spans)
{
Assert.Equal("elasticsearch.query", span.Name);
Assert.Equal("Samples.Elasticsearch.V5-elasticsearch", span.Service);
Assert.Equal("elasticsearch", span.Type);
}
ValidateSpans(spans, (span) => span.Resource, expected);
}
}
public void SubmitsTraces()
{
int agentPort = TcpPortProvider.GetOpenPort();
using (var agent = new MockTracerAgent(agentPort))
using (var processResult = RunSampleAndWaitForExit(agent.Port, arguments: $"StackExchange {TestPrefix}"))
{
Assert.True(processResult.ExitCode >= 0, $"Process exited with code {processResult.ExitCode}");
var expected = new TupleList <string, string>
{
{ "SET", $"SET {TestPrefix}StackExchange.Redis.INCR" },
{ "PING", "PING" },
{ "DDCUSTOM", "DDCUSTOM" },
{ "ECHO", "ECHO" },
{ "SLOWLOG", "SLOWLOG" },
{ "INCR", $"INCR {TestPrefix}StackExchange.Redis.INCR" },
{ "INCRBYFLOAT", $"INCRBYFLOAT {TestPrefix}StackExchange.Redis.INCR" },
{ "GET", $"GET {TestPrefix}StackExchange.Redis.INCR" },
{ "TIME", "TIME" },
};
var batchPrefix = $"{TestPrefix}StackExchange.Redis.Batch.";
expected.AddRange(new TupleList <string, string>
{
{ "DEBUG", $"DEBUG {batchPrefix}DebugObjectAsync" },
{ "DDCUSTOM", $"DDCUSTOM" },
{ "GEOADD", $"GEOADD {batchPrefix}GeoAddAsync" },
{ "GEODIST", $"GEODIST {batchPrefix}GeoDistanceAsync" },
{ "GEOHASH", $"GEOHASH {batchPrefix}GeoHashAsync" },
{ "GEOPOS", $"GEOPOS {batchPrefix}GeoPositionAsync" },
{ "GEORADIUSBYMEMBER", $"GEORADIUSBYMEMBER {batchPrefix}GeoRadiusAsync" },
{ "ZREM", $"ZREM {batchPrefix}GeoRemoveAsync" },
{ "HINCRBYFLOAT", $"HINCRBYFLOAT {batchPrefix}HashDecrementAsync" },
{ "HDEL", $"HDEL {batchPrefix}HashDeleteAsync" },
{ "HEXISTS", $"HEXISTS {batchPrefix}HashExistsAsync" },
{ "HGETALL", $"HGETALL {batchPrefix}HashGetAllAsync" },
{ "HINCRBYFLOAT", $"HINCRBYFLOAT {batchPrefix}HashIncrementAsync" },
{ "HKEYS", $"HKEYS {batchPrefix}HashKeysAsync" },
{ "HLEN", $"HLEN {batchPrefix}HashLengthAsync" },
{ "HMSET", $"HMSET {batchPrefix}HashSetAsync" },
{ "HVALS", $"HVALS {batchPrefix}HashValuesAsync" },
{ "PFADD", $"PFADD {batchPrefix}HyperLogLogAddAsync" },
{ "PFCOUNT", $"PFCOUNT {batchPrefix}HyperLogLogLengthAsync" },
{ "PFMERGE", $"PFMERGE {batchPrefix}HyperLogLogMergeAsync" },
{ "PING", $"PING" },
// { "DEL", $"DEL key" },
{ "DUMP", $"DUMP key" },
{ "EXISTS", $"EXISTS key" },
{ "PEXPIREAT", $"PEXPIREAT key" },
{ "MOVE", $"MOVE key" },
{ "PERSIST", $"PERSIST key" },
{ "RANDOMKEY", $"RANDOMKEY" },
{ "RENAME", "RENAME key1" },
{ "RESTORE", "RESTORE key" },
{ "TYPE", "TYPE key" },
{ "LINDEX", "LINDEX listkey" },
{ "LINSERT", "LINSERT listkey" },
{ "LINSERT", "LINSERT listkey" },
{ "LPOP", "LPOP listkey" },
{ "LPUSH", "LPUSH listkey" },
{ "LLEN", "LLEN listkey" },
{ "LRANGE", "LRANGE listkey" },
{ "LREM", "LREM listkey" },
{ "RPOP", "RPOP listkey" },
{ "RPOPLPUSH", "RPOPLPUSH listkey" },
{ "RPUSH", "RPUSH listkey" },
{ "LSET", "LSET listkey" },
{ "LTRIM", "LTRIM listkey" },
{ "GET", "GET listkey" },
{ "SET", "SET listkey" },
{ "PUBLISH", "PUBLISH channel" },
{ "SADD", "SADD setkey" },
{ "SUNIONSTORE", "SUNIONSTORE setkey" },
{ "SUNION", "SUNION setkey1" },
{ "SISMEMBER", "SISMEMBER setkey" },
{ "SCARD", "SCARD setkey" },
{ "SMEMBERS", "SMEMBERS setkey" },
{ "SMOVE", "SMOVE setkey1" },
{ "SPOP", "SPOP setkey1" },
{ "SRANDMEMBER", "SRANDMEMBER setkey" },
{ "SRANDMEMBER", "SRANDMEMBER setkey" },
{ "SREM", "SREM setkey" },
{ "SORT", "SORT setkey" },
{ "SORT", "SORT setkey" },
{ "ZADD", "ZADD ssetkey" },
{ "ZUNIONSTORE", "ZUNIONSTORE ssetkey1" },
{ "ZINCRBY", "ZINCRBY ssetkey" },
{ "ZINCRBY", "ZINCRBY ssetkey" },
{ "ZCARD", "ZCARD ssetkey" },
{ "ZLEXCOUNT", "ZLEXCOUNT ssetkey" },
{ "ZRANGE", "ZRANGE ssetkey" },
{ "ZRANGE", "ZRANGE ssetkey" },
{ "ZRANGEBYSCORE", "ZRANGEBYSCORE ssetkey" },
{ "ZRANGEBYSCORE", "ZRANGEBYSCORE ssetkey" },
{ "ZRANGEBYLEX", "ZRANGEBYLEX ssetkey" },
{ "ZRANK", "ZRANK ssetkey" },
{ "ZREM", "ZREM ssetkey" },
{ "ZREMRANGEBYRANK", "ZREMRANGEBYRANK ssetkey" },
{ "ZREMRANGEBYSCORE", "ZREMRANGEBYSCORE ssetkey" },
{ "ZREMRANGEBYLEX", "ZREMRANGEBYLEX ssetkey" },
{ "ZSCORE", "ZSCORE ssestkey" },
{ "APPEND", "APPEND ssetkey" },
{ "BITCOUNT", "BITCOUNT ssetkey" },
{ "BITOP", "BITOP" },
{ "BITPOS", "BITPOS ssetkey1" },
{ "INCRBYFLOAT", "INCRBYFLOAT key" },
{ "GET", "GET key" },
{ "GETBIT", "GETBIT key" },
{ "GETRANGE", "GETRANGE key" },
{ "GETSET", "GETSET key" },
{ "INCR", "INCR key" },
{ "STRLEN", "STRLEN key" },
{ "SET", "SET key" },
{ "SETBIT", "SETBIT key" },
{ "SETRANGE", "SETRANGE key" },
});
var dbPrefix = $"{TestPrefix}StackExchange.Redis.Database.";
expected.AddRange(new TupleList <string, string>
{
{ "DEBUG", $"DEBUG {dbPrefix}DebugObject" },
{ "DDCUSTOM", $"DDCUSTOM" },
{ "GEOADD", $"GEOADD {dbPrefix}Geo" },
{ "GEODIST", $"GEODIST {dbPrefix}Geo" },
{ "GEOHASH", $"GEOHASH {dbPrefix}Geo" },
{ "GEOPOS", $"GEOPOS {dbPrefix}Geo" },
{ "GEORADIUSBYMEMBER", $"GEORADIUSBYMEMBER {dbPrefix}Geo" },
{ "ZREM", $"ZREM {dbPrefix}Geo" },
{ "HINCRBYFLOAT", $"HINCRBYFLOAT {dbPrefix}Hash" },
{ "HDEL", $"HDEL {dbPrefix}Hash" },
{ "HEXISTS", $"HEXISTS {dbPrefix}Hash" },
{ "HGET", $"HGET {dbPrefix}Hash" },
{ "HGETALL", $"HGETALL {dbPrefix}Hash" },
{ "HINCRBY", $"HINCRBY {dbPrefix}Hash" },
{ "HKEYS", $"HKEYS {dbPrefix}Hash" },
{ "HLEN", $"HLEN {dbPrefix}Hash" },
// { "HSCAN", $"HSCAN {dbPrefix}Hash" },
{ "HMSET", $"HMSET {dbPrefix}Hash" },
{ "HVALS", $"HVALS {dbPrefix}Hash" },
{ "PFADD", $"PFADD {dbPrefix}HyperLogLog" },
{ "PFCOUNT", $"PFCOUNT {dbPrefix}HyperLogLog" },
{ "PFMERGE", $"PFMERGE {dbPrefix}HyperLogLog2" },
// { "DEL", $"DEL {dbPrefix}Key" },
{ "DUMP", $"DUMP {dbPrefix}Key" },
{ "EXISTS", $"EXISTS {dbPrefix}Key" },
{ "PEXPIREAT", $"PEXPIREAT {dbPrefix}Key" },
{ "MIGRATE", $"MIGRATE {dbPrefix}Key" },
{ "MOVE", $"MOVE {dbPrefix}Key" },
{ "PERSIST", $"PERSIST {dbPrefix}Key" },
{ "RANDOMKEY", $"RANDOMKEY" },
{ "RENAME", $"RENAME {dbPrefix}Key" },
{ "RESTORE", $"RESTORE {dbPrefix}Key" },
{ "PTTL", $"PTTL {dbPrefix}Key" },
{ "TYPE", $"TYPE {dbPrefix}Key" },
{ "LINDEX", $"LINDEX {dbPrefix}List" },
{ "LINSERT", $"LINSERT {dbPrefix}List" },
{ "LINSERT", $"LINSERT {dbPrefix}List" },
{ "LPOP", $"LPOP {dbPrefix}List" },
{ "LPUSH", $"LPUSH {dbPrefix}List" },
{ "LLEN", $"LLEN {dbPrefix}List" },
{ "LRANGE", $"LRANGE {dbPrefix}List" },
{ "LREM", $"LREM {dbPrefix}List" },
{ "RPOP", $"RPOP {dbPrefix}List" },
{ "RPOPLPUSH", $"RPOPLPUSH {dbPrefix}List" },
{ "RPUSH", $"RPUSH {dbPrefix}List" },
{ "LSET", $"LSET {dbPrefix}List" },
{ "LTRIM", $"LTRIM {dbPrefix}List" },
{ "GET", $"GET {dbPrefix}Lock" },
{ "SET", $"SET {dbPrefix}Lock" },
{ "PING", $"PING" },
{ "PUBLISH", $"PUBLISH value" },
{ "SADD", $"SADD {dbPrefix}Set" },
{ "SUNION", $"SUNION {dbPrefix}Set" },
{ "SUNIONSTORE", $"SUNIONSTORE {dbPrefix}Set" },
{ "SISMEMBER", $"SISMEMBER {dbPrefix}Set" },
{ "SCARD", $"SCARD {dbPrefix}Set" },
{ "SMEMBERS", $"SMEMBERS {dbPrefix}Set" },
{ "SMOVE", $"SMOVE {dbPrefix}Set" },
{ "SPOP", $"SPOP {dbPrefix}Set" },
{ "SRANDMEMBER", $"SRANDMEMBER {dbPrefix}Set" },
{ "SRANDMEMBER", $"SRANDMEMBER {dbPrefix}Set" },
{ "SREM", $"SREM {dbPrefix}Set" },
{ "EXEC", $"EXEC" },
{ "SORT", $"SORT {dbPrefix}Key" },
{ "SORT", $"SORT {dbPrefix}Key" },
{ "ZADD", $"ZADD {dbPrefix}SortedSet" },
{ "ZUNIONSTORE", $"ZUNIONSTORE {dbPrefix}SortedSet2" },
{ "ZINCRBY", $"ZINCRBY {dbPrefix}SortedSet" },
{ "ZINCRBY", $"ZINCRBY {dbPrefix}SortedSet" },
{ "ZCARD", $"ZCARD {dbPrefix}SortedSet" },
{ "ZLEXCOUNT", $"ZLEXCOUNT {dbPrefix}SortedSet" },
{ "ZRANGE", $"ZRANGE {dbPrefix}SortedSet" },
{ "ZRANGE", $"ZRANGE {dbPrefix}SortedSet" },
{ "ZRANGEBYSCORE", $"ZRANGEBYSCORE {dbPrefix}SortedSet" },
{ "ZRANGEBYSCORE", $"ZRANGEBYSCORE {dbPrefix}SortedSet" },
{ "ZRANGEBYLEX", $"ZRANGEBYLEX {dbPrefix}SortedSet" },
{ "ZRANK", $"ZRANK {dbPrefix}SortedSet" },
{ "ZREM", $"ZREM {dbPrefix}SortedSet" },
{ "ZREMRANGEBYRANK", $"ZREMRANGEBYRANK {dbPrefix}SortedSet" },
{ "ZREMRANGEBYSCORE", $"ZREMRANGEBYSCORE {dbPrefix}SortedSet" },
{ "ZREMRANGEBYLEX", $"ZREMRANGEBYLEX {dbPrefix}SortedSet" },
{ "ZSCORE", $"ZSCORE {dbPrefix}SortedSet" },
{ "APPEND", $"APPEND {dbPrefix}Key" },
{ "BITCOUNT", $"BITCOUNT {dbPrefix}Key" },
{ "BITOP", $"BITOP" },
{ "BITPOS", $"BITPOS {dbPrefix}Key" },
{ "INCRBYFLOAT", $"INCRBYFLOAT {dbPrefix}Key" },
{ "GET", $"GET {dbPrefix}Key" },
{ "GETBIT", $"GETBIT {dbPrefix}Key" },
{ "GETRANGE", $"GETRANGE {dbPrefix}Key" },
{ "GETSET", $"GETSET {dbPrefix}Key" },
{ "PTTL+GET", $"PTTL+GET {dbPrefix}Key" },
{ "STRLEN", $"STRLEN {dbPrefix}Key" },
{ "SET", $"SET {dbPrefix}Key" },
{ "SETBIT", $"SETBIT {dbPrefix}Key" },
{ "SETRANGE", $"SETRANGE {dbPrefix}Key" },
});
var spans = agent.WaitForSpans(expected.Count).Where(s => s.Type == "redis").OrderBy(s => s.Start).ToList();
var host = Environment.GetEnvironmentVariable("REDIS_HOST") ?? "localhost";
foreach (var span in spans)
{
Assert.Equal("redis.command", span.Name);
Assert.Equal("Samples.RedisCore-redis", span.Service);
Assert.Equal(SpanTypes.Redis, span.Type);
Assert.Equal(host, span.Tags.Get <string>("out.host"));
Assert.Equal("6379", span.Tags.Get <string>("out.port"));
}
var spanLookup = new Dictionary <Tuple <string, string>, int>();
foreach (var span in spans)
{
var key = new Tuple <string, string>(span.Resource, span.Tags.Get <string>("redis.raw_command"));
if (spanLookup.ContainsKey(key))
{
spanLookup[key]++;
}
else
{
spanLookup[key] = 1;
}
}
var missing = new List <Tuple <string, string> >();
foreach (var e in expected)
{
var found = spanLookup.ContainsKey(e);
if (found)
{
if (--spanLookup[e] <= 0)
{
spanLookup.Remove(e);
}
}
else
{
missing.Add(e);
}
}
foreach (var e in missing)
{
Assert.True(false, $"no span found for `{e.Item1}`, `{e.Item2}`, remaining spans: `{string.Join(", ", spanLookup.Select(kvp => $"{kvp.Key.Item1}, {kvp.Key.Item2}").ToArray())}`");
}
}
}
public void SubmitsTraces(bool enableCallTarget)
{
SetCallTargetSettings(enableCallTarget);
int agentPort = TcpPortProvider.GetOpenPort();
int aspNetCorePort = TcpPortProvider.GetOpenPort();
using (var agent = new MockTracerAgent(agentPort))
using (Process process = StartSample(agent.Port, arguments: null, packageVersion: string.Empty, aspNetCorePort: aspNetCorePort))
{
var wh = new EventWaitHandle(false, EventResetMode.AutoReset);
process.OutputDataReceived += (sender, args) =>
{
if (args.Data != null)
{
if (args.Data.Contains("Now listening on:") || args.Data.Contains("Unable to start Kestrel"))
{
wh.Set();
}
Output.WriteLine($"[webserver][stdout] {args.Data}");
}
};
process.BeginOutputReadLine();
process.ErrorDataReceived += (sender, args) =>
{
if (args.Data != null)
{
Output.WriteLine($"[webserver][stderr] {args.Data}");
}
};
process.BeginErrorReadLine();
wh.WaitOne(5000);
var maxMillisecondsToWait = 15_000;
var intervalMilliseconds = 500;
var intervals = maxMillisecondsToWait / intervalMilliseconds;
var serverReady = false;
// wait for server to be ready to receive requests
while (intervals-- > 0)
{
var aliveCheckRequest = new RequestInfo()
{
HttpMethod = "GET", Url = "/alive-check"
};
try
{
serverReady = SubmitRequest(aspNetCorePort, aliveCheckRequest, false) == HttpStatusCode.OK;
}
catch
{
// ignore
}
if (serverReady)
{
Output.WriteLine("The server is ready.");
break;
}
Thread.Sleep(intervalMilliseconds);
}
if (!serverReady)
{
throw new Exception("Couldn't verify the application is ready to receive requests.");
}
var testStart = DateTime.Now;
SubmitRequests(aspNetCorePort);
var graphQLValidateSpans = agent.WaitForSpans(_expectedGraphQLValidateSpanCount, operationName: _graphQLValidateOperationName, returnAllOperations: false)
.GroupBy(s => s.SpanId)
.Select(grp => grp.First())
.OrderBy(s => s.Start);
var graphQLExecuteSpans = agent.WaitForSpans(_expectedGraphQLExecuteSpanCount, operationName: _graphQLExecuteOperationName, returnAllOperations: false)
.GroupBy(s => s.SpanId)
.Select(grp => grp.First())
.OrderBy(s => s.Start);
if (!process.HasExited)
{
process.Kill();
}
var spans = graphQLValidateSpans.Concat(graphQLExecuteSpans).ToList();
SpanTestHelpers.AssertExpectationsMet(_expectations, spans);
}
}
protected async Task AssertWebServerSpan(
string path,
MockTracerAgent agent,
int httpPort,
HttpStatusCode expectedHttpStatusCode,
bool isError,
string expectedAspNetErrorType,
string expectedAspNetErrorMessage,
string expectedErrorType,
string expectedErrorMessage,
string expectedSpanType,
string expectedOperationName,
string expectedAspNetResourceName,
string expectedResourceName,
string expectedServiceVersion,
SerializableDictionary expectedTags = null)
{
IImmutableList <MockTracerAgent.Span> spans;
using (var httpClient = new HttpClient())
{
// disable tracing for this HttpClient request
httpClient.DefaultRequestHeaders.Add(HttpHeaderNames.TracingEnabled, "false");
var testStart = DateTime.UtcNow;
var response = await httpClient.GetAsync($"http://localhost:{httpPort}" + path);
var content = await response.Content.ReadAsStringAsync();
Output.WriteLine($"[http] {response.StatusCode} {content}");
Assert.Equal(expectedHttpStatusCode, response.StatusCode);
agent.SpanFilters.Add(IsServerSpan);
spans = agent.WaitForSpans(
count: 2,
minDateTime: testStart,
returnAllOperations: true);
Assert.True(spans.Count == 2, $"expected two span, saw {spans.Count}");
}
MockTracerAgent.Span aspnetSpan = spans.Where(s => s.Name == "aspnet.request").FirstOrDefault();
MockTracerAgent.Span innerSpan = spans.Where(s => s.Name == expectedOperationName).FirstOrDefault();
Assert.NotNull(aspnetSpan);
Assert.Equal(expectedAspNetResourceName, aspnetSpan.Resource);
Assert.NotNull(innerSpan);
Assert.Equal(expectedResourceName, innerSpan.Resource);
foreach (MockTracerAgent.Span span in spans)
{
// base properties
Assert.Equal(expectedSpanType, span.Type);
// errors
Assert.Equal(isError, span.Error == 1);
if (span == aspnetSpan)
{
Assert.Equal(expectedAspNetErrorType, span.Tags.GetValueOrDefault(Tags.ErrorType));
Assert.Equal(expectedAspNetErrorMessage, span.Tags.GetValueOrDefault(Tags.ErrorMsg));
}
else if (span == innerSpan)
{
Assert.Equal(expectedErrorType, span.Tags.GetValueOrDefault(Tags.ErrorType));
Assert.Equal(expectedErrorMessage, span.Tags.GetValueOrDefault(Tags.ErrorMsg));
}
// other tags
Assert.Equal(SpanKinds.Server, span.Tags.GetValueOrDefault(Tags.SpanKind));
Assert.Equal(expectedServiceVersion, span.Tags.GetValueOrDefault(Tags.Version));
}
if (expectedTags?.Values is not null)
{
foreach (var expectedTag in expectedTags)
{
Assert.Equal(expectedTag.Value, innerSpan.Tags.GetValueOrDefault(expectedTag.Key));
}
}
}
public void SubmitTraces()
{
SetCallTargetSettings(true);
const int expectedTransactionalTraces = 13;
const int expectedNonTransactionalTracesTraces = 12;
const int totalTransactions = expectedTransactionalTraces + expectedNonTransactionalTracesTraces;
const int expectedPurgeCount = 2;
const int expectedPeekCount = 3;
const int expectedSendCount = 10;
const int expectedReceiveCount = 10;
var sendCount = 0;
var peekCount = 0;
var receiveCount = 0;
var purgeCount = 0;
var transactionalTraces = 0;
var nonTransactionalTraces = 0;
var agentPort = TcpPortProvider.GetOpenPort();
using var agent = new MockTracerAgent(agentPort);
using var processResult = RunSampleAndWaitForExit(agent.Port, arguments: $"5 5");
Assert.True(processResult.ExitCode >= 0, $"Process exited with code {processResult.ExitCode} and exception: {processResult.StandardError}");
var spans = agent.WaitForSpans(totalTransactions);
Assert.True(spans.Count >= totalTransactions, $"Expecting at least {totalTransactions} spans, only received {spans.Count}");
var msmqSpans = spans.Where(span => string.Equals(span.Service, ExpectedServiceName, StringComparison.OrdinalIgnoreCase));
foreach (var span in msmqSpans)
{
span.Type.Should().Be(SpanTypes.Queue);
span.Service.Should().Be(ExpectedServiceName);
span.Tags.Should().Contain(new System.Collections.Generic.KeyValuePair <string, string>(Tags.InstrumentationName, "msmq"));
if (span.Tags[Tags.MsmqIsTransactionalQueue] == "True")
{
span.Tags[Tags.MsmqQueuePath].Should().Be(".\\Private$\\private-transactional-queue");
transactionalTraces++;
}
else
{
span.Tags[Tags.MsmqQueuePath].Should().Be(".\\Private$\\private-nontransactional-queue");
nonTransactionalTraces++;
}
span.Name.Should().Be("msmq.command");
span.Tags?.ContainsKey(Tags.Version).Should().BeFalse("External service span should not have service version tag.");
var command = span.Tags[Tags.MsmqCommand];
if (string.Equals(command, "msmq.send", StringComparison.OrdinalIgnoreCase))
{
span.Tags[Tags.MsmqMessageWithTransaction].Should().Be(span.Tags[Tags.MsmqIsTransactionalQueue], "The program is supposed to send messages within transactions to transactional queues, and outside of transactions to non transactional queues");
span.Tags[Tags.SpanKind].Should().Be(SpanKinds.Producer);
span.Resource.Should().Be($"msmq.send {span.Tags[Tags.MsmqQueuePath]}");
sendCount++;
}
else if (string.Equals(command, "msmq.receive", StringComparison.OrdinalIgnoreCase))
{
span.Tags[Tags.SpanKind].Should().Be(SpanKinds.Consumer);
span.Resource.Should().Be($"msmq.receive {span.Tags[Tags.MsmqQueuePath]}");
receiveCount++;
}
else if (string.Equals(command, "msmq.peek", StringComparison.OrdinalIgnoreCase))
{
span.Tags[Tags.SpanKind].Should().Be(SpanKinds.Consumer);
span.Resource.Should().Be($"msmq.peek {span.Tags[Tags.MsmqQueuePath]}");
peekCount++;
}
else if (string.Equals(command, "msmq.purge", StringComparison.OrdinalIgnoreCase))
{
span.Tags[Tags.SpanKind].Should().Be(SpanKinds.Client);
span.Resource.Should().Be($"msmq.purge {span.Tags[Tags.MsmqQueuePath]}");
purgeCount++;
}
else
{
throw new Xunit.Sdk.XunitException($"msmq.command {command} not recognized.");
}
}
nonTransactionalTraces.Should().Be(expectedNonTransactionalTracesTraces);
transactionalTraces.Should().Be(expectedTransactionalTraces);
sendCount.Should().Be(expectedSendCount);
purgeCount.Should().Be(expectedPurgeCount);
receiveCount.Should().Be(expectedReceiveCount);
peekCount.Should().Be(expectedPeekCount);
}
public void SubmitsTraces(string packageVersion, bool enableCallTarget)
{
SetCallTargetSettings(enableCallTarget);
var expectedSpanCount = 26;
int basicPublishCount = 0;
int basicGetCount = 0;
int basicDeliverCount = 0;
int exchangeDeclareCount = 0;
int queueDeclareCount = 0;
int queueBindCount = 0;
var distributedParentSpans = new Dictionary <ulong, int>();
int emptyBasicGetCount = 0;
int agentPort = TcpPortProvider.GetOpenPort();
using (var agent = new MockTracerAgent(agentPort))
using (var processResult = RunSampleAndWaitForExit(agent.Port, arguments: $"{TestPrefix}", packageVersion: packageVersion))
{
Assert.True(processResult.ExitCode >= 0, $"Process exited with code {processResult.ExitCode} and exception: {processResult.StandardError}");
var spans = agent.WaitForSpans(expectedSpanCount); // Do not filter on operation name because they will vary depending on instrumented method
Assert.True(spans.Count >= expectedSpanCount, $"Expecting at least {expectedSpanCount} spans, only received {spans.Count}");
var rabbitmqSpans = spans.Where(span => string.Equals(span.Service, ExpectedServiceName, StringComparison.OrdinalIgnoreCase));
var manualSpans = spans.Where(span => !string.Equals(span.Service, ExpectedServiceName, StringComparison.OrdinalIgnoreCase));
foreach (var span in rabbitmqSpans)
{
Assert.Equal(SpanTypes.Queue, span.Type);
Assert.Equal("RabbitMQ", span.Tags[Tags.InstrumentationName]);
Assert.False(span.Tags?.ContainsKey(Tags.Version), "External service span should not have service version tag.");
Assert.NotNull(span.Tags[Tags.AmqpCommand]);
Assert.Equal("amqp.command", span.Name);
var command = span.Tags[Tags.AmqpCommand];
if (command.StartsWith("basic.", StringComparison.OrdinalIgnoreCase))
{
if (string.Equals(command, "basic.publish", StringComparison.OrdinalIgnoreCase))
{
basicPublishCount++;
Assert.Equal(SpanKinds.Producer, span.Tags[Tags.SpanKind]);
Assert.NotNull(span.Tags[Tags.AmqpExchange]);
Assert.NotNull(span.Tags[Tags.AmqpRoutingKey]);
Assert.NotNull(span.Tags["message.size"]);
Assert.True(int.TryParse(span.Tags["message.size"], out _));
// Enforce that the resource name has the following structure: "basic.publish [<default>|{actual exchangeName}] -> [<all>|<generated>|{actual routingKey}]"
string regexPattern = @"basic\.publish (?<exchangeName>\S*) -> (?<routingKey>\S*)";
var match = Regex.Match(span.Resource, regexPattern);
Assert.True(match.Success);
var exchangeName = match.Groups["exchangeName"].Value;
Assert.True(string.Equals(exchangeName, "<default>") || string.Equals(exchangeName, span.Tags[Tags.AmqpExchange]));
var routingKey = match.Groups["routingKey"].Value;
Assert.True(string.Equals(routingKey, "<all>") || string.Equals(routingKey, "<generated>") || string.Equals(routingKey, span.Tags[Tags.AmqpRoutingKey]));
}
else if (string.Equals(command, "basic.get", StringComparison.OrdinalIgnoreCase))
{
basicGetCount++;
// Successful responses will have the "message.size" tag
// Empty responses will not
if (span.Tags.TryGetValue("message.size", out string messageSize))
{
Assert.NotNull(span.ParentId);
Assert.True(int.TryParse(messageSize, out _));
// Add the parent span ID to a dictionary so we can later assert 1:1 mappings
if (distributedParentSpans.TryGetValue(span.ParentId.Value, out int count))
{
distributedParentSpans[span.ParentId.Value] = count + 1;
}
else
{
distributedParentSpans[span.ParentId.Value] = 1;
}
}
else
{
emptyBasicGetCount++;
}
Assert.Equal(SpanKinds.Consumer, span.Tags[Tags.SpanKind]);
Assert.NotNull(span.Tags[Tags.AmqpQueue]);
// Enforce that the resource name has the following structure: "basic.get [<generated>|{actual queueName}]"
string regexPattern = @"basic\.get (?<queueName>\S*)";
var match = Regex.Match(span.Resource, regexPattern);
Assert.True(match.Success);
var queueName = match.Groups["queueName"].Value;
Assert.True(string.Equals(queueName, "<generated>") || string.Equals(queueName, span.Tags[Tags.AmqpQueue]));
}
else if (string.Equals(command, "basic.deliver", StringComparison.OrdinalIgnoreCase))
{
basicDeliverCount++;
Assert.NotNull(span.ParentId);
// Add the parent span ID to a dictionary so we can later assert 1:1 mappings
if (distributedParentSpans.TryGetValue(span.ParentId.Value, out int count))
{
distributedParentSpans[span.ParentId.Value] = count + 1;
}
else
{
distributedParentSpans[span.ParentId.Value] = 1;
}
Assert.Equal(SpanKinds.Consumer, span.Tags[Tags.SpanKind]);
// Assert.NotNull(span.Tags[Tags.AmqpQueue]); // Java does this but we're having difficulty doing this. Push to v2?
Assert.NotNull(span.Tags[Tags.AmqpExchange]);
Assert.NotNull(span.Tags[Tags.AmqpRoutingKey]);
Assert.NotNull(span.Tags["message.size"]);
Assert.True(int.TryParse(span.Tags["message.size"], out _));
// Enforce that the resource name has the following structure: "basic.deliver [<generated>|{actual queueName}]"
string regexPattern = @"basic\.deliver (?<queueName>\S*)";
var match = Regex.Match(span.Resource, regexPattern);
// Assert.True(match.Success); // Enable once we can get the queue name included
var queueName = match.Groups["queueName"].Value;
// Assert.True(string.Equals(queueName, "<generated>") || string.Equals(queueName, span.Tags[Tags.AmqpQueue])); // Enable once we can get the queue name included
}
else
{
throw new Xunit.Sdk.XunitException($"amqp.command {command} not recognized.");
}
}
else
{
Assert.Equal(SpanKinds.Client, span.Tags[Tags.SpanKind]);
Assert.Equal(command, span.Resource);
if (string.Equals(command, "exchange.declare", StringComparison.OrdinalIgnoreCase))
{
exchangeDeclareCount++;
Assert.NotNull(span.Tags[Tags.AmqpExchange]);
}
else if (string.Equals(command, "queue.declare", StringComparison.OrdinalIgnoreCase))
{
queueDeclareCount++;
Assert.NotNull(span.Tags[Tags.AmqpQueue]);
}
else if (string.Equals(command, "queue.bind", StringComparison.OrdinalIgnoreCase))
{
queueBindCount++;
Assert.NotNull(span.Tags[Tags.AmqpExchange]);
Assert.NotNull(span.Tags[Tags.AmqpQueue]);
Assert.NotNull(span.Tags[Tags.AmqpRoutingKey]);
}
else
{
throw new Xunit.Sdk.XunitException($"amqp.command {command} not recognized.");
}
}
}
foreach (var span in manualSpans)
{
Assert.Equal("Samples.RabbitMQ", span.Service);
Assert.Equal("1.0.0", span.Tags[Tags.Version]);
}
}
// Assert that all empty get results are expected
Assert.Equal(2, emptyBasicGetCount);
// Assert that each span that started a distributed trace (basic.publish)
// has only one child span (basic.deliver or basic.get)
Assert.All(distributedParentSpans, kvp => Assert.Equal(1, kvp.Value));
Assert.Equal(5, basicPublishCount);
Assert.Equal(4, basicGetCount);
Assert.Equal(3, basicDeliverCount);
Assert.Equal(1, exchangeDeclareCount);
Assert.Equal(1, queueBindCount);
Assert.Equal(4, queueDeclareCount);
}
public void SubmitsTraces(string packageVersion, bool enableCallTarget)
{
SetCallTargetSettings(enableCallTarget);
int agentPort = TcpPortProvider.GetOpenPort();
using (var agent = new MockTracerAgent(agentPort))
using (var processResult = RunSampleAndWaitForExit(agent.Port, arguments: $"{TestPrefix}", packageVersion: packageVersion))
{
Assert.True(processResult.ExitCode >= 0, $"Process exited with code {processResult.ExitCode}");
// note: ignore the INFO command because it's timing is unpredictable (on Linux?)
var spans = agent.WaitForSpans(11)
.Where(s => s.Type == "redis" && s.Resource != "INFO" && s.Resource != "ROLE" && s.Resource != "QUIT")
.OrderBy(s => s.Start)
.ToList();
var host = Environment.GetEnvironmentVariable("SERVICESTACK_REDIS_HOST") ?? "localhost:6379";
var port = host.Substring(host.IndexOf(':') + 1);
host = host.Substring(0, host.IndexOf(':'));
foreach (var span in spans)
{
Assert.Equal("redis.command", span.Name);
Assert.Equal("Samples.ServiceStack.Redis-redis", span.Service);
Assert.Equal(SpanTypes.Redis, span.Type);
Assert.Equal(host, DictionaryExtensions.GetValueOrDefault(span.Tags, "out.host"));
Assert.Equal(port, DictionaryExtensions.GetValueOrDefault(span.Tags, "out.port"));
Assert.False(span.Tags?.ContainsKey(Tags.Version), "External service span should not have service version tag.");
}
var expectedFromOneRun = new TupleList <string, string>
{
{ "SET", $"SET {TestPrefix}ServiceStack.Redis.INCR 0" },
{ "PING", "PING" },
{ "DDCUSTOM", "DDCUSTOM COMMAND" },
{ "ECHO", "ECHO Hello World" },
{ "SLOWLOG", "SLOWLOG GET 5" },
{ "INCR", $"INCR {TestPrefix}ServiceStack.Redis.INCR" },
{ "INCRBYFLOAT", $"INCRBYFLOAT {TestPrefix}ServiceStack.Redis.INCR 1.25" },
{ "TIME", "TIME" },
{ "SELECT", "SELECT 0" },
};
var expected = new TupleList <string, string>();
expected.AddRange(expectedFromOneRun);
expected.AddRange(expectedFromOneRun);
#if NETCOREAPP3_1 || NET5_0
expected.AddRange(expectedFromOneRun); // On .NET Core 3.1 and .NET 5 we run the routine a third time
#endif
Assert.Equal(expected.Count, spans.Count);
for (int i = 0; i < expected.Count; i++)
{
var e1 = expected[i].Item1;
var e2 = expected[i].Item2;
var a1 = i < spans.Count
? spans[i].Resource
: string.Empty;
var a2 = i < spans.Count
? DictionaryExtensions.GetValueOrDefault(spans[i].Tags, "redis.raw_command")
: string.Empty;
Assert.True(e1 == a1, $@"invalid resource name for span #{i}, expected ""{e1}"", actual ""{a1}""");
Assert.True(e2 == a2, $@"invalid raw command for span #{i}, expected ""{e2}"" != ""{a2}""");
}
}
}
public void SubmitsTraces(string packageVersion)
{
var topic = $"sample-topic-{TestPrefix}-{packageVersion}".Replace('.', '-');
var agentPort = TcpPortProvider.GetOpenPort();
using var agent = new MockTracerAgent(agentPort);
using var processResult = RunSampleAndWaitForExit(agent.Port, arguments: topic, packageVersion: packageVersion);
processResult.ExitCode.Should().BeGreaterOrEqualTo(0);
var allSpans = agent.WaitForSpans(TotalExpectedSpanCount, timeoutInMilliseconds: 10_000);
using var assertionScope = new AssertionScope();
// We use HaveCountGreaterOrEqualTo because _both_ consumers may handle the message
// Due to manual/autocommit behaviour
allSpans.Should().HaveCountGreaterOrEqualTo(TotalExpectedSpanCount);
var allProducerSpans = allSpans.Where(x => x.Name == "kafka.produce").ToList();
var successfulProducerSpans = allProducerSpans.Where(x => x.Error == 0).ToList();
var errorProducerSpans = allProducerSpans.Where(x => x.Error > 0).ToList();
var allConsumerSpans = allSpans.Where(x => x.Name == "kafka.consume").ToList();
VerifyProducerSpanProperties(successfulProducerSpans, GetSuccessfulResourceName("Produce", topic), ExpectedSuccessProducerSpans + ExpectedTombstoneProducerSpans);
VerifyProducerSpanProperties(errorProducerSpans, ErrorProducerResourceName, ExpectedErrorProducerSpans);
// Only successful spans with a delivery handler will have an offset
successfulProducerSpans
.Where(span => span.Tags.ContainsKey(Tags.KafkaOffset))
.Select(span => span.Tags[Tags.KafkaOffset])
.Should()
.OnlyContain(tag => Regex.IsMatch(tag, @"^[0-9]+$"))
.And.HaveCount(ExpectedSuccessProducerWithHandlerSpans + ExpectedTombstoneProducerWithHandlerSpans);
// Only successful spans with a delivery handler will have a partition
// Confirm partition is displayed correctly [0], [1]
// https://github.com/confluentinc/confluent-kafka-dotnet/blob/master/src/Confluent.Kafka/Partition.cs#L217-L224
successfulProducerSpans
.Where(span => span.Tags.ContainsKey(Tags.KafkaPartition))
.Select(span => span.Tags[Tags.KafkaPartition])
.Should()
.OnlyContain(tag => Regex.IsMatch(tag, @"^\[[0-9]+\]$"))
.And.HaveCount(ExpectedSuccessProducerWithHandlerSpans + ExpectedTombstoneProducerWithHandlerSpans);
allProducerSpans
.Where(span => span.Tags.ContainsKey(Tags.KafkaTombstone))
.Select(span => span.Tags[Tags.KafkaTombstone])
.Should()
.HaveCount(ExpectedTombstoneProducerSpans)
.And.OnlyContain(tag => tag == "true");
// verify have error
errorProducerSpans.Should().OnlyContain(x => x.Tags.ContainsKey(Tags.ErrorType))
.And.ContainSingle(x => x.Tags[Tags.ErrorType] == "Confluent.Kafka.ProduceException`2[System.String,System.String]") // created by async handler
.And.ContainSingle(x => x.Tags[Tags.ErrorType] == "System.Exception"); // created by sync callback handler
var producerSpanIds = successfulProducerSpans
.Select(x => x.SpanId)
.Should()
.OnlyHaveUniqueItems()
.And.Subject.ToImmutableHashSet();
VerifyConsumerSpanProperties(allConsumerSpans, GetSuccessfulResourceName("Consume", topic), ExpectedConsumerSpans);
// every consumer span should be a child of a producer span.
allConsumerSpans
.Should()
.OnlyContain(span => span.ParentId.HasValue)
.And.OnlyContain(span => producerSpanIds.Contains(span.ParentId.Value));
// HaveCountGreaterOrEqualTo because same message may be consumed by both
allConsumerSpans
.Where(span => span.Tags.ContainsKey(Tags.KafkaTombstone))
.Select(span => span.Tags[Tags.KafkaTombstone])
.Should()
.HaveCountGreaterOrEqualTo(ExpectedTombstoneProducerSpans)
.And.OnlyContain(tag => tag == "true");
}
