public async Task GrpcInputTests_StopsAndRestarts()
{
// ARRANGE
int instancesReceived = 0;
InstanceMsg receivedInstance = null;
int port = Common.GetPort();
var input = new GrpcInput("localhost", port, instance =>
{
instancesReceived++;
receivedInstance = instance;
});
input.Start();
Assert.IsTrue(SpinWait.SpinUntil(() => input.IsRunning, GrpcInputTests.DefaultTimeout));
var grpcWriter = new GrpcWriter(port);
var request = new HandleTraceSpanRequest();
request.Instances.Add(new InstanceMsg()
{
SourceName = "SourceName1"
});
await grpcWriter.Write(request).ConfigureAwait(false);
Common.AssertIsTrueEventually(
() => input.GetStats().InstancesReceived == 1 && instancesReceived == 1 &&
receivedInstance.SourceName == "SourceName1", GrpcInputTests.DefaultTimeout);
// ACT
input.Stop();
Common.AssertIsTrueEventually(
() => !input.IsRunning && input.GetStats().InstancesReceived == 1 && instancesReceived == 1 &&
receivedInstance.SourceName == "SourceName1", GrpcInputTests.DefaultTimeout);
input.Start();
Assert.IsTrue(SpinWait.SpinUntil(() => input.IsRunning, GrpcInputTests.DefaultTimeout));
grpcWriter = new GrpcWriter(port);
request.Instances.Single().SourceName = "SourceName2";
await grpcWriter.Write(request).ConfigureAwait(false);
// ASSERT
Common.AssertIsTrueEventually(
() => input.IsRunning && input.GetStats().InstancesReceived == 1 && instancesReceived == 2 &&
receivedInstance.SourceName == "SourceName2", GrpcInputTests.DefaultTimeout);
}
private void OnDataReceived(InstanceMsg instance)
{
try
{
foreach (var telemetry in this.telemetryGenerator.Generate(instance))
{
if (telemetry != null)
{
this.telemetryClient.Track(telemetry);
}
}
}
catch (System.Exception e)
{
// unexpected exception occured, let the caller handle it
throw;
}
}
private void OnDataReceived(InstanceMsg instance)
{
try
{
foreach (var telemetry in this.telemetryGenerator.Generate(instance))
{
if (telemetry != null)
{
this.telemetryClient.Track(telemetry);
}
}
}
catch (System.Exception e)
{
// unexpected exception occured
Diagnostics.LogError(FormattableString.Invariant($"Unknown exception while processing an instance. {e.ToString()}"));
}
}
public void GrpcInputTests_ReceivesDataFromMultipleClients()
{
// ARRANGE
int instancesReceived = 0;
InstanceMsg receivedInstance = null;
int port = Common.GetPort();
var input = new GrpcInput("localhost", port, instance =>
{
Interlocked.Increment(ref instancesReceived);
receivedInstance = instance;
});
input.Start();
Assert.IsTrue(SpinWait.SpinUntil(() => input.IsRunning, GrpcInputTests.DefaultTimeout));
// ACT
var request = new HandleTraceSpanRequest();
request.Instances.Add(new InstanceMsg()
{
SourceName = "SourceName1"
});
Parallel.For(0, 1000, new ParallelOptions()
{
MaxDegreeOfParallelism = 1000
}, async i =>
{
var grpcWriter = new GrpcWriter(port);
await grpcWriter.Write(request).ConfigureAwait(false);
});
// ASSERT
Common.AssertIsTrueEventually(
() => input.GetStats().InstancesReceived == 1000 && instancesReceived == 1000, GrpcInputTests.DefaultTimeout);
input.Stop();
Assert.IsTrue(SpinWait.SpinUntil(() => !input.IsRunning, GrpcInputTests.DefaultTimeout));
}
/// <summary>
/// Generates a set of AI telemetry based on a Mixer instance
/// </summary>
/// <param name="instance"></param>
/// <returns></returns>
public IEnumerable <ITelemetry> Generate(InstanceMsg instance)
{
#if DEBUG_INFO
var debugInfo = instance.ToString();
Diagnostics.LogInfo($"Received instance: {debugInfo}");
#endif
var sourceUid = instance.SpanTags["source.uid"].StringValue;
var sourceName = instance.SpanTags["source.name"].StringValue;
var sourceWorkloadName = instance.SpanTags["source.workload.name"].StringValue;
var sourceWorkloadNamespace = instance.SpanTags["source.workload.namespace"].StringValue;
var destinationUid = instance.SpanTags["destination.uid"].StringValue;
var destinationName = instance.SpanTags["destination.name"].StringValue;
var destinationWorkloadName = instance.SpanTags["destination.workload.name"].StringValue;
var destinationWorkloadNamespace = instance.SpanTags["destination.workload.namespace"].StringValue;
var contextReporterUid = instance.SpanTags["context.reporter.uid"].StringValue;
var contextReporterKind = instance.SpanTags["context.reporter.kind"].StringValue.ToLowerInvariant();
var contextProtocol = instance.SpanTags["context.protocol"].StringValue.ToLowerInvariant();
//var connectionEvent = instance.SpanTags["connection.event"].StringValue.ToLowerInvariant();
//!!! configure the adapter to receive less trace spans through configuration (rules, etc), not filter them here. That would save bandwidth and mixer CPU
var sourceAppInsightsMonitoringEnabled = instance.SpanTags["source.labels.appinsights.monitoring.enabled"].StringValue;
var destinationAppInsightsMonitoringEnabled = instance.SpanTags["destination.labels.appinsights.monitoring.enabled"].StringValue;
var isSourceIstioIngressGateway = instance.SpanTags["source.labels.istio.isingressgateway"].BoolValue;
var requestHeadersRequestId = instance.SpanTags["request.headers.request.id"].StringValue;
var requestHeadersSyntheticTestRunId = instance.SpanTags["request.headers.synthetictest.runid"].StringValue;
var requestHeadersSyntheticTestLocation = instance.SpanTags["request.headers.synthetictest.location"].StringValue;
var requestHeadersRequestContext = instance.SpanTags["request.headers.request.context"].StringValue;
var responseHeadersRequestContext = instance.SpanTags["response.headers.request.context"].StringValue;
string requestHeadersRequestContextAppId = string.Empty;
if (!string.IsNullOrWhiteSpace(requestHeadersRequestContext))
{
// Regex.Match is thread-safe
Match requestHeadersRequestContextAppIdMatch = requestContextAppIdRegex.Match(requestHeadersRequestContext);
Group group = requestHeadersRequestContextAppIdMatch.Groups["appId"];
if (requestHeadersRequestContextAppIdMatch.Success && group.Success && group.Captures.Count == 1)
{
requestHeadersRequestContextAppId = requestHeadersRequestContextAppIdMatch.Groups["appId"].Value;
}
}
string responseHeadersRequestContextAppId = string.Empty;
if (!string.IsNullOrWhiteSpace(responseHeadersRequestContext))
{
// Regex.Match is thread-safe
Match responseHeadersRequestContextAppIdMatch = requestContextAppIdRegex.Match(responseHeadersRequestContext);
Group group = responseHeadersRequestContextAppIdMatch.Groups["appId"];
if (responseHeadersRequestContextAppIdMatch.Success && group.Success && group.Captures.Count == 1)
{
responseHeadersRequestContextAppId = responseHeadersRequestContextAppIdMatch.Groups["appId"].Value;
}
}
var sourceRoleName = instance.SpanTags["source.role.name"].StringValue.ToLowerInvariant().Replace("kubernetes://", string.Empty);
var sourceRoleInstance = instance.SpanTags["source.role.instance"].StringValue.ToLowerInvariant().Replace("kubernetes://", string.Empty);
var destinationRoleName = instance.SpanTags["destination.role.name"].StringValue.ToLowerInvariant().Replace("kubernetes://", string.Empty);
var destinationRoleInstance = instance.SpanTags["destination.role.instance"].StringValue.ToLowerInvariant().Replace("kubernetes://", string.Empty);
var httpUserAgent = instance.SpanTags["http.useragent"].StringValue;
var host = instance.SpanTags["host"].StringValue;
var httpStatusCode = instance.SpanTags["http.status_code"].Int64Value;
var httpPath = instance.SpanTags["http.path"].StringValue;
var httpMethod = instance.SpanTags["http.method"].StringValue;
var requestScheme = instance.SpanTags["request.scheme"].StringValue;
var destinationPort = instance.SpanTags["destination.port"].Int64Value;
var destinatonServiceUid = instance.SpanTags["destination.service.uid"].StringValue;
var destinatonServiceHost = instance.SpanTags["destination.service.host"].StringValue;
var destinatonServiceName = instance.SpanTags["destination.service.name"].StringValue;
var destinatonServiceNamespace = instance.SpanTags["destination.service.namespace"].StringValue;
// if source/desination information is insufficient, let's assume it's an external caller/dependency and use whatever little we know about them
if (string.IsNullOrWhiteSpace(sourceRoleName) || string.Equals(sourceRoleName, UnknownValue, StringComparison.InvariantCultureIgnoreCase))
{
sourceRoleName = httpUserAgent;
}
if (string.IsNullOrWhiteSpace(sourceRoleInstance) || string.Equals(sourceRoleInstance, UnknownValue, StringComparison.InvariantCultureIgnoreCase))
{
sourceRoleInstance = httpUserAgent;
}
if (string.IsNullOrWhiteSpace(destinationRoleName) || string.Equals(destinationRoleName, UnknownValue, StringComparison.InvariantCultureIgnoreCase))
{
destinationRoleName = host;
}
if (string.IsNullOrWhiteSpace(destinationRoleInstance) || string.Equals(destinationRoleInstance, UnknownValue, StringComparison.InvariantCultureIgnoreCase))
{
destinationRoleInstance = host;
}
// if none of the workloads have anything to do with us, skip the instance completely
this.DetermineInterest(contextReporterUid, sourceWorkloadNamespace, destinationWorkloadNamespace, sourceAppInsightsMonitoringEnabled, destinationAppInsightsMonitoringEnabled,
out bool isInstanceInteresting, out bool isInstanceFullyWithinTargetArea, out bool isSourceWithinTargetArea, out bool isDestinationWithinTargetArea);
Diagnostics.LogTrace($"source: {sourceName}.{sourceWorkloadNamespace}, destination: {destinationName}.{destinationWorkloadNamespace}, reporter: {contextReporterUid}, kind: {contextReporterKind}");
//if (contextProtocol == "tcp")
//{
// Diagnostics.LogTrace(Invariant($"TCP. {debugInfo}"));
//}
if (!isInstanceInteresting || !contextProtocol.StartsWith("http") /*|| (contextProtocol == "tcp" && connectionEvent != "open")*/)
{
Diagnostics.LogTrace($"SKIPPED: isInstanceInteresting: {isInstanceInteresting}, contextProtocol: {contextProtocol}");
yield break;
}
// Request-Id header format https://github.com/dotnet/corefx/blob/master/src/System.Diagnostics.DiagnosticSource/src/HierarchicalRequestId.md
bool incomingRequestIdPresent = !string.IsNullOrWhiteSpace(requestHeadersRequestId);
string incomingRequestId = incomingRequestIdPresent ? requestHeadersRequestId : Invariant($"|{RandomStringLong()}.");
string url = Invariant(
$@"{contextProtocol}{(string.IsNullOrWhiteSpace(contextProtocol) ? string.Empty : "://")}{destinationRoleInstance}{httpPath}{
(destinationPort > 0 ? $":{destinationPort}" : string.Empty)
}");
long code = httpStatusCode == 0 ? 0 : (httpStatusCode >= 400 ? httpStatusCode : 0);
// this instance signifies a dependency call (client span) for the source and a request (server span) for the destination
// from the distributed tracing perspective, it's dependency call => request
// we need to report both
// each instance where both workloads are within our target area of interest will be reported twice - by the two proxies it passes through
// for these calls, we'll arbitrarily choose to act only on the instance supplied by the inbound proxy to avoid duplication
// each instance where the source is within our target area and the destination is not may be reported twice
// for these calls, we'll only act on the outbound proxy
// each instance where the source is not within our target area and the destination is may be reported twice
// for these calls, we'll only act on the inbound proxy
bool instanceIsActionable = isInstanceFullyWithinTargetArea && string.Equals(contextReporterKind, "inbound", StringComparison.InvariantCultureIgnoreCase) ||
isSourceWithinTargetArea && !isDestinationWithinTargetArea && string.Equals(contextReporterKind, "outbound", StringComparison.InvariantCultureIgnoreCase) ||
!isSourceWithinTargetArea && isDestinationWithinTargetArea && string.Equals(contextReporterKind, "inbound", StringComparison.InvariantCultureIgnoreCase);
if (!instanceIsActionable)
{
Diagnostics.LogTrace($"NOT ACTIONABLE: isInstanceFullyWithinTargetArea: {isInstanceFullyWithinTargetArea}, contextReporterKind: {contextReporterKind}, isSourceWithinTargetArea: {isSourceWithinTargetArea}, isDestinationWithinTargetArea: {isDestinationWithinTargetArea}");
yield break;
}
var debugData = new Dictionary <string, string>()
{
{ "k8s.context.reporter.kind", contextReporterKind },
{ "k8s.context.reporter.uid", contextReporterUid },
{ "k8s.source.uid", sourceUid },
{ "k8s.source.name", sourceName },
{ "k8s.source.workload.name", sourceWorkloadName },
{ "k8s.source.workload.namespace", sourceWorkloadNamespace },
{ "k8s.destination.uid", destinationUid },
{ "k8s.destination.name", destinationName },
{ "k8s.destination.workload.name", destinationWorkloadName },
{ "k8s.destination.workload.namespace", destinationWorkloadNamespace },
{ "k8s.destination.service.uid", destinatonServiceUid },
{ "k8s.destination.service.host", destinatonServiceHost },
{ "k8s.destination.service.name", destinatonServiceName },
{ "k8s.destination.service.namespace", destinatonServiceNamespace },
#if DEBUG_INFO
{ "aks.debug.info", debugInfo }
#endif
};
DateTimeOffset?startTime = instance.StartTime?.Value?.ToDateTimeOffset();
DateTimeOffset?endTime = instance.EndTime?.Value?.ToDateTimeOffset();
string latestRequestId = incomingRequestId;
var log = new StringBuilder();
// if the source is an Istio Ingress Gateway we want to generate an additional pair of dependency/request on its behalf
// as if it's within our target area (we want to see it on the App Map)
if (isSourceIstioIngressGateway)
{
log.Append(" RD(G) ");
// we are generating a request/dependency pair for the Istio Ingress Gateway
// the instance represents the call: gateway -> pod (reported by inbound proxy of the pod)
// on behalf of the gateway, report server span (request)
// the request will represent the gateway's side of the call: internet -> gateway
string gatewayRequestId = incomingRequestIdPresent ? AcknowledgeRequest(incomingRequestId) : incomingRequestId;
yield return(this.GenerateRequest(spanName: destinationRoleInstance,
requestId: gatewayRequestId,
parentRequestId: incomingRequestIdPresent ? incomingRequestId : string.Empty,
spanStatusCode: code, spanStatusMessage: string.Empty, startTime: startTime, endTime: endTime,
roleName: sourceRoleName,
roleInstance: sourceRoleInstance,
httpUrl: url, httpHost: host,
httpStatusCode: httpStatusCode.ToString(CultureInfo.InvariantCulture), httpPath: httpPath, httpMethod: httpMethod, httpPort: destinationPort, httpScheme: requestScheme,
httpUserAgent: httpUserAgent, httpRoute: string.Empty, requestHeadersRequestContextAppId: requestHeadersRequestContextAppId, requestHeadersSyntheticTestRunId: requestHeadersSyntheticTestRunId,
requestHeadersSyntheticTestLocation: requestHeadersSyntheticTestLocation, propagateSyntheticContext: true, debugData: debugData));
// on behalf of the Gateway, report client span (dependency call)
// the dependency will represent the gateway's side of the call: gateway -> pod
string requestId = StartDependency(gatewayRequestId);
yield return(this.GenerateDependency(spanName: destinationRoleInstance,
requestId: requestId,
parentRequestId: gatewayRequestId,
spanStatusCode: code, spanStatusMessage: string.Empty, startTime: startTime,
endTime: endTime,
roleName: sourceRoleName,
roleInstance: sourceRoleInstance,
httpUrl: url,
httpHost: host, httpStatusCode: httpStatusCode.ToString(CultureInfo.InvariantCulture), httpPath: httpPath, httpMethod: httpMethod, httpPort: destinationPort, httpScheme: requestScheme,
protocol: contextProtocol, responseHeadersRequestContextAppId: responseHeadersRequestContextAppId,
debugData: debugData));
// advance latestRequestId so that the main dependency/request pair is stiched to the ingress gateway pair we've already created
latestRequestId = requestId;
}
// now, let's generate the regular pair of dependency/request
// first, on behalf of the source, report client span (dependency call)
bool isIngressRequest = !isSourceWithinTargetArea && isDestinationWithinTargetArea;
if (!isIngressRequest)
{
log.Append(" D ");
string requestId = StartDependency(latestRequestId);
//!!! Temporary support of app propagating Request-Id
bool isEgressRequest = isSourceWithinTargetArea && !isDestinationWithinTargetArea;
if (isEgressRequest && incomingRequestIdPresent)
{
requestId = incomingRequestId;
}
/////////// end of temporary support
yield return(this.GenerateDependency(spanName: destinationRoleInstance,
requestId: requestId,
parentRequestId: latestRequestId,
spanStatusCode: code, spanStatusMessage: string.Empty, startTime: startTime, endTime: endTime,
roleName: sourceRoleName,
roleInstance: sourceRoleInstance,
httpUrl: url,
httpHost: host, httpStatusCode: httpStatusCode.ToString(CultureInfo.InvariantCulture), httpPath: httpPath, httpMethod: httpMethod, httpPort: destinationPort, httpScheme: requestScheme,
protocol: contextProtocol, responseHeadersRequestContextAppId: responseHeadersRequestContextAppId,
debugData: debugData));
// advance latestRequestId
latestRequestId = requestId;
}
// now, on behalf of the destination, report server span (request), but only if the destination is within the target area
// otherwise it's an external dependency and we don't want to generate requests for it
if (isDestinationWithinTargetArea)
{
log.Append(" R ");
// we only want to propagate synthetic context if this is a direct call from a synthetic probe (not through a gateway)
// if it's a synthetic call through a gateway, the gateway will be the one for which we've propagated synthetic context above
yield return(this.GenerateRequest(spanName: destinationRoleInstance,
requestId: AcknowledgeRequest(latestRequestId),
parentRequestId: latestRequestId,
spanStatusCode: code, spanStatusMessage: string.Empty, startTime: startTime, endTime: endTime,
roleName: destinationRoleName,
roleInstance: destinationRoleInstance,
httpUrl: url, httpHost: host,
httpStatusCode: httpStatusCode.ToString(CultureInfo.InvariantCulture), httpPath: httpPath, httpMethod: httpMethod, httpPort: destinationPort, httpScheme: requestScheme,
httpUserAgent: httpUserAgent, httpRoute: string.Empty, requestHeadersRequestContextAppId: requestHeadersRequestContextAppId, requestHeadersSyntheticTestRunId: requestHeadersSyntheticTestRunId,
requestHeadersSyntheticTestLocation: requestHeadersSyntheticTestLocation, propagateSyntheticContext: !isSourceIstioIngressGateway, debugData: debugData));
}
Diagnostics.LogTrace(log.ToString());
}
