/// <summary>
/// Processes an incoming telemetry batch for OpenCensus channel.
/// </summary>
/// <remarks>This method may be called from multiple threads concurrently.</remarks>
private void OnOcBatchReceived(ExportTraceServiceRequest batch, ServerCallContext callContext)
{
try
{
TryGetOrUpdatePeerInfo(batch.Node, callContext, out Node peerInfo);
// send incoming telemetry items to the telemetryClient
foreach (Span span in batch.Spans)
{
try
{
Diagnostics.LogTrace($"OpenCensus message received: {batch.Spans.Count} spans, first span: {batch.Spans.FirstOrDefault()?.Name}");
this.telemetryClient.TrackSpan(span, peerInfo, this.ocToAiInstrumentationKey);
}
catch (Exception e)
{
// an unexpected issue while tracking an item
// log and carry on
Diagnostics.LogError(
FormattableString.Invariant(
$"Could not track an incoming OpenCensus telemetry item. {e.ToString()}"));
}
}
}
catch (Exception e)
{
// an unexpected issue while processing the batch
// log and carry on
Diagnostics.LogError(
FormattableString.Invariant(
$"Could not process an incoming OpenCensus telemetry batch. {e.ToString()}"));
}
}
public async Task HostTests_StopsLibrary()
{
// ARRANGE
var telemetryClient = Common.SetupStubTelemetryClient(out var sentItems);
int portAI = Common.GetPort();
int portOC = Common.GetPort();
var config = $@"<?xml version=""1.0"" encoding=""utf-8"" ?>
<LocalForwarderConfiguration>
<Inputs>
<ApplicationInsightsInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portAI}</Port>
</ApplicationInsightsInput>
<OpenCensusInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portOC}</Port>
</OpenCensusInput>
</Inputs>
<OpenCensusToApplicationInsights>
<InstrumentationKey>ikey1</InstrumentationKey>
</OpenCensusToApplicationInsights>
</LocalForwarderConfiguration>
";
Host host = new Host(telemetryClient);
host.Run(config, TimeSpan.FromSeconds(5));
Thread.Sleep(TimeSpan.FromMilliseconds(250));
// ACT
host.Stop();
Thread.Sleep(TimeSpan.FromMilliseconds(250));
var telemetryBatch = new ExportTraceServiceRequest();
telemetryBatch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span1"
}, Kind = Span.Types.SpanKind.Server
});
telemetryBatch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span2"
}, Kind = Span.Types.SpanKind.Client
});
// ASSERT
var writer = new GrpcWriter(false, portOC);
await writer.Write(telemetryBatch).ConfigureAwait(false);
Assert.Fail();
}
private async Task RunAsync(CancellationToken cancellationToken)
{
var duplexCall = this.traceClient.Export();
try
{
bool firstRequest = true;
while (!cancellationToken.IsCancellationRequested)
{
var spanExportRequest = new ExportTraceServiceRequest();
if (firstRequest)
{
spanExportRequest.Node = this.node;
}
// Spans
bool hasSpans = false;
while (spanExportRequest.Spans.Count < this.spanBatchSize)
{
if (!this.spans.TryDequeue(out var spanData))
{
break;
}
var protoSpan = spanData.ToProtoSpan();
if (protoSpan == null)
{
continue;
}
spanExportRequest.Spans.Add(protoSpan);
hasSpans = true;
}
if (hasSpans)
{
await duplexCall.RequestStream.WriteAsync(spanExportRequest).ConfigureAwait(false);
firstRequest = false;
}
else
{
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false);
}
}
await duplexCall.RequestStream.CompleteAsync().ConfigureAwait(false);
}
catch (RpcException)
{
// TODO: log
throw;
}
}
public void GrpcOpenCensusInputTests_ReceivesDataFromMultipleClients()
{
// ARRANGE
int batchesReceived = 0;
ExportTraceServiceRequest receivedBatch = null;
int port = GetPort();
var input = new GrpcOpenCensusInput("localhost", port);
input.Start(
(exportSpanRequest, callContext) =>
{
Interlocked.Increment(ref batchesReceived);
receivedBatch = exportSpanRequest;
},
(configRequest, callContext) => ConfigResponse);
Assert.IsTrue(SpinWait.SpinUntil(() => input.IsRunning, GrpcOpenCensusInputTests.DefaultTimeout));
// ACT
ExportTraceServiceRequest batch = new ExportTraceServiceRequest();
batch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Event1"
}
});
Parallel.For(0, 1000, new ParallelOptions()
{
MaxDegreeOfParallelism = 1000
}, async i =>
{
var grpcWriter = new GrpcWriter(false, port);
await grpcWriter.Write(batch).ConfigureAwait(false);
});
// ASSERT
Common.AssertIsTrueEventually(
() => input.GetStats().BatchesReceived == 1000 && batchesReceived == 1000, GrpcOpenCensusInputTests.DefaultTimeout);
input.Stop();
Assert.IsTrue(SpinWait.SpinUntil(() => !input.IsRunning, GrpcOpenCensusInputTests.DefaultTimeout));
}
public async Task GrpcOpenCensusInputTests_ReceivesSpansWithNode()
{
// ARRANGE
int batchesReceived = 0;
ExportTraceServiceRequest receivedBatch = null;
int port = GetPort();
var input = new GrpcOpenCensusInput("localhost", port);
input.Start(
(telemetryBatch, callContext) =>
{
batchesReceived++;
receivedBatch = telemetryBatch;
},
null);
Assert.IsTrue(SpinWait.SpinUntil(() => input.IsRunning, GrpcOpenCensusInputTests.DefaultTimeout));
var grpcWriter = new GrpcWriter(false, port);
// ACT
ExportTraceServiceRequest batch = new ExportTraceServiceRequest();
batch.Spans.Add(new Span {
Name = new TruncatableString {
Value = "Event1"
}
});
batch.Node = new Node {
Identifier = new ProcessIdentifier {
Pid = 2
}
};
await grpcWriter.Write(batch).ConfigureAwait(false);
// ASSERT
Common.AssertIsTrueEventually(
() => input.GetStats().BatchesReceived == 1 && batchesReceived == 1 &&
receivedBatch.Node.Identifier.Pid == 2 &&
receivedBatch.Spans.Single().Name.Value == "Event1",
GrpcOpenCensusInputTests.DefaultTimeout);
input.Stop();
Assert.IsTrue(SpinWait.SpinUntil(() => !input.IsRunning, GrpcOpenCensusInputTests.DefaultTimeout));
}
public async Task Write(ExportTraceServiceRequest batch)
{
if (this.aiMode)
{
throw new InvalidOperationException("Incorrect mode");
}
try
{
await this.openCensusExportStreamingCall.RequestStream.WriteAsync(batch).ConfigureAwait(false);
}
catch (System.Exception e)
{
throw new InvalidOperationException(
FormattableString.Invariant($"Error sending a message via gRpc. {e.ToString()}"));
}
}
public async Task GrpcOpenCensusInputTests_StopsWhileWaitingForData()
{
// ARRANGE
int batchesReceived = 0;
ExportTraceServiceRequest receivedBatch = null;
int port = GetPort();
var input = new GrpcOpenCensusInput("localhost", port);
input.Start(
(exportSpanRequest, callContext) =>
{
batchesReceived++;
receivedBatch = exportSpanRequest;
},
(configRequest, callContext) => ConfigResponse);
Assert.IsTrue(SpinWait.SpinUntil(() => input.IsRunning, GrpcOpenCensusInputTests.DefaultTimeout));
var grpcWriter = new GrpcWriter(false, port);
ExportTraceServiceRequest batch = new ExportTraceServiceRequest();
batch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Event1"
}
});
await grpcWriter.Write(batch).ConfigureAwait(false);
Common.AssertIsTrueEventually(
() => input.GetStats().BatchesReceived == 1 && batchesReceived == 1 &&
receivedBatch.Spans.Single().Name.Value == "Event1", GrpcOpenCensusInputTests.DefaultTimeout);
// ACT
input.Stop();
// ASSERT
Common.AssertIsTrueEventually(
() => !input.IsRunning && input.GetStats().BatchesReceived == 1 && batchesReceived == 1 &&
receivedBatch.Spans.Single().Name.Value == "Event1", GrpcOpenCensusInputTests.DefaultTimeout);
}
public async Task GrpcOpenCensusInputTests_HandlesExceptionsInSpanRequestsProcessingHandler()
{
// ARRANGE
int port = GetPort();
var input = new GrpcOpenCensusInput("localhost", port);
input.Start((exportSpanRequest, callContext) => throw new InvalidOperationException(), (configRequest, callContext) => ConfigResponse);
Assert.IsTrue(SpinWait.SpinUntil(() => input.IsRunning, GrpcOpenCensusInputTests.DefaultTimeout));
var grpcWriter = new GrpcWriter(false, port);
ExportTraceServiceRequest batch = new ExportTraceServiceRequest();
batch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Event1"
}
});
// ACT
await grpcWriter.Write(batch).ConfigureAwait(false);
// ASSERT
// must have handled the exception by logging it
// should still be able to process items
Common.AssertIsTrueEventually(
() => input.IsRunning && input.GetStats().BatchesReceived == 0 && input.GetStats().BatchesFailed == 1,
GrpcOpenCensusInputTests.DefaultTimeout);
await grpcWriter.Write(batch).ConfigureAwait(false);
Common.AssertIsTrueEventually(
() => input.IsRunning && input.GetStats().BatchesReceived == 0 && input.GetStats().BatchesFailed == 2,
GrpcOpenCensusInputTests.DefaultTimeout);
}
public async Task HostTests_RunsLibrary()
{
// ARRANGE
var telemetryClient = Common.SetupStubTelemetryClient(out var sentItems);
int portAI = Common.GetPort();
int portOC = Common.GetPort();
var config = $@"<?xml version=""1.0"" encoding=""utf-8"" ?>
<LocalForwarderConfiguration>
<Inputs>
<ApplicationInsightsInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portAI}</Port>
</ApplicationInsightsInput>
<OpenCensusInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portOC}</Port>
</OpenCensusInput>
</Inputs>
<OpenCensusToApplicationInsights>
<InstrumentationKey>ikey1</InstrumentationKey>
</OpenCensusToApplicationInsights>
<ApplicationInsights>
<LiveMetricsStreamInstrumentationKey>ikey1</LiveMetricsStreamInstrumentationKey>
<LiveMetricsStreamAuthenticationApiKey></LiveMetricsStreamAuthenticationApiKey>
<AdaptiveSampling Enabled=""false"">
<MaxEventsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGEVENTSLIMIT%</MaxEventsPerSecond>
<!--Telemetry items other than events are counted together-->
<MaxOtherItemsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGNONEVENTSLIMIT%</MaxOtherItemsPerSecond>
</AdaptiveSampling>
</ApplicationInsights>
</LocalForwarderConfiguration>
";
// ACT
Host host = new Host(telemetryClient);
host.Run(config, TimeSpan.FromSeconds(5));
Thread.Sleep(TimeSpan.FromMilliseconds(250));
var telemetryBatch = new ExportTraceServiceRequest();
telemetryBatch.Spans.Add(new Span {
Name = new TruncatableString {
Value = "Span1"
}, Kind = Span.Types.SpanKind.Server
});
telemetryBatch.Spans.Add(new Span {
Name = new TruncatableString {
Value = "Span2"
}, Kind = Span.Types.SpanKind.Client
});
telemetryBatch.Node = new Node {
ServiceInfo = new ServiceInfo {
Name = "tests"
}
};
var configRequest = new CurrentLibraryConfig
{
Config = new TraceConfig
{
RateLimitingSampler = new RateLimitingSampler {
Qps = 1
}
}
};
var writer = new GrpcWriter(false, portOC);
await writer.Write(telemetryBatch).ConfigureAwait(false);
await writer.Write(configRequest).ConfigureAwait(false);
Common.AssertIsTrueEventually(() => sentItems.Count == 3);
// ASSERT
Assert.AreEqual("tests", (sentItems.Skip(0).First() as EventTelemetry).Context.Cloud.RoleName);
Assert.AreEqual("Span1", (sentItems.Skip(1).First() as RequestTelemetry).Name);
Assert.AreEqual("Span2", (sentItems.Skip(2).First() as DependencyTelemetry).Name);
}
public async Task HostTests_RestartsLibraryIfStoppedUnexpectedly()
{
// ARRANGE
var telemetryClient = Common.SetupStubTelemetryClient(out var sentItems);
int portAI = Common.GetPort();
int portOC = Common.GetPort();
var config = $@"<?xml version=""1.0"" encoding=""utf-8"" ?>
<LocalForwarderConfiguration>
<Inputs>
<ApplicationInsightsInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portAI}</Port>
</ApplicationInsightsInput>
<OpenCensusInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portOC}</Port>
</OpenCensusInput>
</Inputs>
<OpenCensusToApplicationInsights>
<InstrumentationKey>ikey1</InstrumentationKey>
</OpenCensusToApplicationInsights>
<ApplicationInsights>
<LiveMetricsStreamInstrumentationKey>ikey1</LiveMetricsStreamInstrumentationKey>
<LiveMetricsStreamAuthenticationApiKey></LiveMetricsStreamAuthenticationApiKey>
<AdaptiveSampling Enabled=""false"">
<MaxEventsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGEVENTSLIMIT%</MaxEventsPerSecond>
<!--Telemetry items other than events are counted together-->
<MaxOtherItemsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGNONEVENTSLIMIT%</MaxOtherItemsPerSecond>
</AdaptiveSampling>
</ApplicationInsights>
</LocalForwarderConfiguration>
";
Host host = new Host(telemetryClient);
host.Run(config, TimeSpan.FromSeconds(1));
FieldInfo libraryFieldInfo = host.GetType().GetField("library", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic);
Common.AssertIsTrueEventually(() => libraryFieldInfo.GetValue(host) != null);
// ACT
// stop the existing library (as if something went wrong)
var library = libraryFieldInfo.GetValue(host) as Library;
library.Stop();
Common.AssertIsTrueEventually(() => !library.IsRunning);
// ASSERT
// wait for a new library
Common.AssertIsTrueEventually(() => libraryFieldInfo.GetValue(host) != null);
Common.AssertIsTrueEventually(() => (libraryFieldInfo.GetValue(host) as Library).IsRunning);
// verify the new library works
var telemetryBatch = new ExportTraceServiceRequest();
telemetryBatch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span1"
}, Kind = Span.Types.SpanKind.Server
});
telemetryBatch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span2"
}, Kind = Span.Types.SpanKind.Client
});
var writer = new GrpcWriter(false, portOC);
await writer.Write(telemetryBatch).ConfigureAwait(false);
Common.AssertIsTrueEventually(() => sentItems.Count == 2);
Assert.AreEqual("Span1", (sentItems.Skip(0).First() as RequestTelemetry).Name);
Assert.AreEqual("Span2", (sentItems.Skip(1).First() as DependencyTelemetry).Name);
}
public async Task LibraryTests_LibraryLogsInputStatsCorrectly()
{
// ARRANGE
var telemetryClient = Common.SetupStubTelemetryClient(out var sentItems);
int portAI = Common.GetPort();
int portOC = Common.GetPort();
var config = $@"<?xml version=""1.0"" encoding=""utf-8"" ?>
<LocalForwarderConfiguration>
<Inputs>
<ApplicationInsightsInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portAI}</Port>
</ApplicationInsightsInput>
<OpenCensusInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portOC}</Port>
</OpenCensusInput>
</Inputs>
<OpenCensusToApplicationInsights>
<InstrumentationKey>ikey1</InstrumentationKey>
</OpenCensusToApplicationInsights>
<ApplicationInsights>
<LiveMetricsStreamInstrumentationKey>[SPECIFY LIVE METRICS STREAM INSTRUMENTATION KEY HERE]</LiveMetricsStreamInstrumentationKey>
<LiveMetricsStreamAuthenticationApiKey></LiveMetricsStreamAuthenticationApiKey>
<AdaptiveSampling Enabled=""true"">
<MaxEventsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGEVENTSLIMIT%</MaxEventsPerSecond>
<!--Telemetry items other than events are counted together-->
<MaxOtherItemsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGNONEVENTSLIMIT%</MaxOtherItemsPerSecond>
</AdaptiveSampling>
</ApplicationInsights>
</LocalForwarderConfiguration>
";
var telemetryBatchAI = new TelemetryBatch();
telemetryBatchAI.Items.Add(new Telemetry()
{
Event = new Event()
{
Name = "Event1"
}
});
telemetryBatchAI.Items.Add(new Telemetry()
{
Message = new Message()
{
Message_ = "Message1"
}
});
telemetryBatchAI.Items.Add(new Telemetry()
{
Metric = new LocalForwarder.Library.Inputs.Contracts.Metric()
{
Metrics = { new DataPoint()
{
Name = "Metric1", Value = 1
} }
}
});
telemetryBatchAI.Items.Add(new Telemetry()
{
Exception = new LocalForwarder.Library.Inputs.Contracts.Exception()
{
ProblemId = "Exception1", Exceptions = { new ExceptionDetails()
{
Message = "Exception1"
} }
}
});
telemetryBatchAI.Items.Add(new Telemetry()
{
Dependency = new Dependency()
{
Name = "Dependency1"
}
});
telemetryBatchAI.Items.Add(new Telemetry()
{
Availability = new Availability()
{
Name = "Availability1"
}
});
telemetryBatchAI.Items.Add(new Telemetry()
{
PageView = new PageView()
{
Id = "PageView1"
}
});
telemetryBatchAI.Items.Add(new Telemetry()
{
Request = new Request()
{
Name = "Request1"
}
});
var telemetryBatchOC = new ExportTraceServiceRequest();
telemetryBatchOC.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span1"
}, Kind = Span.Types.SpanKind.Server
});
telemetryBatchOC.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span2"
}, Kind = Span.Types.SpanKind.Client
});
// redirect loggging to a new file
Diagnostics.Flush(TimeSpan.FromSeconds(5));
string logFileName = Common.SwitchLoggerToDifferentFile();
var lib = new Library(config, telemetryClient, TimeSpan.FromMilliseconds(10));
lib.Run();
// ACT
var writer = new GrpcWriter(true, portAI);
await writer.Write(telemetryBatchAI).ConfigureAwait(false);
writer = new GrpcWriter(false, portOC);
await writer.Write(telemetryBatchOC).ConfigureAwait(false);
// ASSERT
Common.AssertIsTrueEventually(() => sentItems.Count == 10);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
lib.Stop();
Diagnostics.Flush(TimeSpan.FromSeconds(5));
// close the file
Common.SwitchLoggerToDifferentFile();
string logs = await File.ReadAllTextAsync(logFileName).ConfigureAwait(false);
Assert.IsTrue(logs.Contains("|INFO|AI input: [ConnectionCount: 0, BatchesReceived: 0, BatchesFailed: 0, ConfigsReceived: 0, ConfigsFailed: 0]"));
Assert.IsTrue(logs.Contains("|INFO|OpenCensus input: [ConnectionCount: 0, BatchesReceived: 0, BatchesFailed: 0, ConfigsReceived: 0, ConfigsFailed: 0]"));
Assert.IsTrue(logs.Contains("|INFO|AI input: [ConnectionCount: 0, BatchesReceived: 1, BatchesFailed: 0, ConfigsReceived: 0, ConfigsFailed: 0]"));
Assert.IsTrue(logs.Contains("|INFO|OpenCensus input: [ConnectionCount: 0, BatchesReceived: 1, BatchesFailed: 0, ConfigsReceived: 0, ConfigsFailed: 0]"));
Assert.IsFalse(logs.Contains("|INFO|AI input: [ConnectionCount: 0, BatchesReceived: 2, BatchesFailed, ConfigsReceived: 0, ConfigsFailed: 0]"));
Assert.IsFalse(logs.Contains("|INFO|OpenCensus input: [ConnectionCount: 0, BatchesReceived: 2, BatchesFailed, ConfigsReceived: 0, ConfigsFailed: 0]"));
}
public async Task LibraryTests_LibraryProcessesOcBatchesCorrectly()
{
// ARRANGE
var telemetryClient = Common.SetupStubTelemetryClient(out var sentItems);
int portAI = Common.GetPort();
int portOC = Common.GetPort();
var config = $@"<?xml version=""1.0"" encoding=""utf-8"" ?>
<LocalForwarderConfiguration>
<Inputs>
<ApplicationInsightsInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portAI}</Port>
</ApplicationInsightsInput>
<OpenCensusInput Enabled=""true"">
<Host>0.0.0.0</Host>
<Port>{portOC}</Port>
</OpenCensusInput>
</Inputs>
<OpenCensusToApplicationInsights>
<InstrumentationKey>ikey1</InstrumentationKey>
</OpenCensusToApplicationInsights>
<ApplicationInsights>
<LiveMetricsStreamInstrumentationKey>[SPECIFY LIVE METRICS STREAM INSTRUMENTATION KEY HERE]</LiveMetricsStreamInstrumentationKey>
<LiveMetricsStreamAuthenticationApiKey></LiveMetricsStreamAuthenticationApiKey>
<AdaptiveSampling Enabled=""true"">
<MaxEventsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGEVENTSLIMIT%</MaxEventsPerSecond>
<!--Telemetry items other than events are counted together-->
<MaxOtherItemsPerSecond>%APPINSIGHTS_ADAPTIVESAMPLINGNONEVENTSLIMIT%</MaxOtherItemsPerSecond>
</AdaptiveSampling>
</ApplicationInsights>
</LocalForwarderConfiguration>
";
var telemetryBatch = new ExportTraceServiceRequest();
telemetryBatch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span1"
}, Kind = Span.Types.SpanKind.Server
});
telemetryBatch.Spans.Add(new Span()
{
Name = new TruncatableString()
{
Value = "Span2"
}, Kind = Span.Types.SpanKind.Client
});
var lib = new Library(config, telemetryClient);
lib.Run();
// ACT
var writer = new GrpcWriter(false, portOC);
await writer.Write(telemetryBatch).ConfigureAwait(false);
// ASSERT
Common.AssertIsTrueEventually(() => sentItems.Count == 2);
lib.Stop();
Assert.AreEqual("Span1", (sentItems.Skip(0).First() as RequestTelemetry).Name);
Assert.AreEqual("Span2", (sentItems.Skip(1).First() as DependencyTelemetry).Name);
}
public override Task <ExportTraceServiceResponse> Export(ExportTraceServiceRequest request, ServerCallContext context)
{
_logger.Log(Microsoft.Extensions.Logging.LogLevel.Information, $"Received call with resource spans count {request.ResourceSpans.Count}");
return(Task.FromResult(new ExportTraceServiceResponse()));
}
