async public Task RetryOn429WithSpecificDate_429HappensOnce()
{
const int delayMs = 10000;
// The actual retry delay will be slightly less than delayMs since UtcNow is recalculated in RetryWithServerDelay()
var errorMargin = TimeSpan.FromMilliseconds(50).TotalMilliseconds;
var actualResponseFromTestRun = new List <Response>();
uint actualDelayFromTestRun = 0;
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithApiKey("123456").WithMaxRetryAttempts(1));
dataSender.WithDelayFunction(async(uint milliSecondsDelay) =>
{
actualDelayFromTestRun = milliSecondsDelay;
await Task.Delay(0);
return;
});
dataSender.WithHttpHandlerImpl((json) =>
{
var httpResponse = new HttpResponseMessage((HttpStatusCode)429);
var retryOnSpecificTime = DateTimeOffset.UtcNow + TimeSpan.FromMilliseconds(delayMs);
httpResponse.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(retryOnSpecificTime);
return(Task.FromResult(httpResponse));
});
var spanBatch = SpanBatchBuilder.Create()
.WithSpan(SpanBuilder.Create("Test Span").Build())
.Build();
var response = await dataSender.SendDataAsync(spanBatch);
Assert.IsTrue(actualDelayFromTestRun >= delayMs - errorMargin && actualDelayFromTestRun <= delayMs + errorMargin, $"Expected delay: {delayMs}, margin: +/-{errorMargin}, actual delay: {actualDelayFromTestRun}");
}
async public Task SendDataAsyncThrowsHttpException()
{
const int expectedNumSendBatchAsyncCall = 1;
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithApiKey("123456"));
dataSender.WithDelayFunction(async(uint milliSecondsDelay) =>
{
await Task.Delay(0);
return;
});
var actualCountCallsSendData = 0;
dataSender.WithCaptureSendDataAsyncDelegate((spanBatch, retryNum) =>
{
actualCountCallsSendData++;
});
dataSender.WithHttpHandlerImpl((json) =>
{
throw new Exception("Server Error", new Exception("Inner exception message"));
});
var spanBatch = SpanBatchBuilder.Create()
.WithSpan(SpanBuilder.Create("Test Span").Build())
.Build();
var result = await dataSender.SendDataAsync(spanBatch);
Assert.AreEqual(NewRelicResponseStatus.Failure, result.ResponseStatus);
Assert.AreEqual("Inner exception message", result.Message);
Assert.IsNull(result.HttpStatusCode);
Assert.AreEqual(expectedNumSendBatchAsyncCall, actualCountCallsSendData, "Unexpected Number of SendDataAsync calls");
}
async public Task RequestTooLarge_SplitFail()
{
const int expectedCountCallsSendData = 7;
const int expectedCountSuccessfulSpanBatches = 1;
const string traceID_Success = "OK";
const string traceID_SplitBatch_Prefix = "TooLarge";
var actualCountCallsSendData = 0;
var successfulSpans = new List <Span>();
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithApiKey("123456"));
var shouldSplitJsons = new List <string>();
// Mock the behavior to return EntityTooLarge for any span batch that has a span with an
// id that starts with TooLarge.
dataSender.WithCaptureSendDataAsyncDelegate((spanBatch, retryNum) =>
{
actualCountCallsSendData++;
if (spanBatch.Spans.Any(x => x.Id.StartsWith(traceID_SplitBatch_Prefix)))
{
shouldSplitJsons.Add(spanBatch.ToJson());
}
else
{
successfulSpans.AddRange(spanBatch.Spans);
}
});
dataSender.WithHttpHandlerImpl((json) =>
{
var response = shouldSplitJsons.Contains(json)
? new HttpResponseMessage(System.Net.HttpStatusCode.RequestEntityTooLarge)
: new HttpResponseMessage(System.Net.HttpStatusCode.OK);
return(Task.FromResult(response));
});
var spanBatchBuilder = SpanBatchBuilder.Create();
spanBatchBuilder.WithSpan(SpanBuilder.Create($"{traceID_SplitBatch_Prefix}1").Build());
spanBatchBuilder.WithSpan(SpanBuilder.Create($"{traceID_SplitBatch_Prefix}2").Build());
spanBatchBuilder.WithSpan(SpanBuilder.Create($"{traceID_SplitBatch_Prefix}3").Build());
spanBatchBuilder.WithSpan(SpanBuilder.Create(traceID_Success).Build());
var spanBatch = spanBatchBuilder.Build();
// Act
var result = await dataSender.SendDataAsync(spanBatch);
// Assert
Assert.AreEqual(NewRelicResponseStatus.Failure, result.ResponseStatus);
Assert.AreEqual(expectedCountCallsSendData, actualCountCallsSendData, "Unexpected number of calls");
Assert.AreEqual(expectedCountSuccessfulSpanBatches, successfulSpans.Count, $"Only {expectedCountSuccessfulSpanBatches} span should have successfully sent");
Assert.AreEqual(traceID_Success, successfulSpans[0].Id, "Incorrect span was sent");
}
async public Task RetryOn429_RetriesExceeded()
{
const int delayMS = 10000;
const int expectedNumSendBatchAsyncCall = 9; // 1 first call + 3 calls from retries
var expectedBackoffSequenceFromTestRun = new List <int>()
{
delayMS,
delayMS,
delayMS,
delayMS,
delayMS,
delayMS,
delayMS,
delayMS
};
var actualBackoffSequenceFromTestRun = new List <uint>();
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithApiKey("123456"));
dataSender.WithDelayFunction(async(uint milliSecondsDelay) =>
{
actualBackoffSequenceFromTestRun.Add(milliSecondsDelay);
await Task.Delay(0);
return;
});
var actualCountCallsSendData = 0;
dataSender.WithCaptureSendDataAsyncDelegate((spanBatch, retryNum) =>
{
actualCountCallsSendData++;
});
dataSender.WithHttpHandlerImpl((json) =>
{
var httpResponse = new HttpResponseMessage((HttpStatusCode)429);
httpResponse.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromMilliseconds(delayMS));
return(Task.FromResult(httpResponse));
});
var spanBatch = SpanBatchBuilder.Create()
.WithSpan(SpanBuilder.Create("Test Span").Build())
.Build();
var result = await dataSender.SendDataAsync(spanBatch);
Assert.AreEqual(NewRelicResponseStatus.Failure, result.ResponseStatus);
Assert.AreEqual(expectedNumSendBatchAsyncCall, actualCountCallsSendData, "Unexpected Number of SendDataAsync calls");
CollectionAssert.AreEqual(expectedBackoffSequenceFromTestRun, actualBackoffSequenceFromTestRun);
}
async public Task RetryBackoffSequence_IntermittentTimeoutEventuallySucceeds()
{
var expectedNumSendBatchAsyncCall = 4; // 1 first call + 3 calls from retries
var expectedBackoffSequenceFromTestRun = new List <int>()
{
5000,
10000,
20000,
};
var actualBackoffSequenceFromTestRun = new List <uint>();
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithApiKey("123456"));
dataSender.WithDelayFunction(async(uint milliSecondsDelay) =>
{
actualBackoffSequenceFromTestRun.Add(milliSecondsDelay);
await Task.Delay(0);
return;
});
var actualCountCallsSendData = 0;
dataSender.WithCaptureSendDataAsyncDelegate((spanBatch, retryNum) =>
{
actualCountCallsSendData++;
});
var callCount = 0;
dataSender.WithHttpHandlerImpl((json) =>
{
callCount++;
if (callCount < 4)
{
return(Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.RequestTimeout)));
}
return(Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK)));
});
var spanBatch = SpanBatchBuilder.Create()
.WithSpan(SpanBuilder.Create("Test Span").Build())
.Build();
var result = await dataSender.SendDataAsync(spanBatch);
Assert.AreEqual(NewRelicResponseStatus.Success, result.ResponseStatus);
Assert.AreEqual(expectedNumSendBatchAsyncCall, actualCountCallsSendData, "Unexpected Number of SendDataAsync calls");
CollectionAssert.AreEqual(expectedBackoffSequenceFromTestRun, actualBackoffSequenceFromTestRun);
}
/// <summary>
/// At startup, read configuration settings and instantiate the data service
/// which will communicate with the New Relic endpoint.
/// </summary>
private static void Configuration()
{
// obtain application settings from appsettings.json file
// Most importantly, the config file will contain the New Relic API Key
var config = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json")
.Build();
var telemetryConfiguration = new TelemetryConfiguration(config);
Console.WriteLine($"{telemetryConfiguration.TraceUrl}");
// The SpanDataSender handles all communication with the New Relic end point
_dataSvc = new SpanDataSender(telemetryConfiguration);
}
async public Task SendDataAsyncThrowsNonHttpException()
{
const int expectedNumSendBatchAsyncCall = 1;
const int expectedNumHttpHandlerCall = 0;
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithAPIKey("123456"));
dataSender.WithDelayFunction(async(int milliSecondsDelay) =>
{
await Task.Delay(0);
return;
});
var actualCountCallsSendData = 0;
dataSender.WithCaptureSendDataAsyncDelegate((sb, retry) =>
{
actualCountCallsSendData++;
throw new Exception("Test Exception");
});
var actualCallsHttpHandler = 0;
dataSender.WithHttpHandlerImpl((json) =>
{
actualCallsHttpHandler++;
return(Task.FromResult(new HttpResponseMessage()
{
StatusCode = HttpStatusCode.OK
}));
});
var spanBatch = SpanBatchBuilder.Create()
.WithSpan(SpanBuilder.Create("Test Span").Build())
.Build();
var result = await dataSender.SendDataAsync(spanBatch);
Assert.AreEqual(NewRelicResponseStatus.Failure, result.ResponseStatus);
Assert.AreEqual("Test Exception", result.Message);
Assert.IsNull(result.HttpStatusCode);
Assert.AreEqual(expectedNumSendBatchAsyncCall, actualCountCallsSendData, "Unexpected Number of SendDataAsync calls");
Assert.AreEqual(expectedNumHttpHandlerCall, actualCallsHttpHandler, "Unexpected Number of Http Handler calls");
}
public void SendAnEmptySpanBatch()
{
var traceId = "123";
var spanBatch = SpanBatchBuilder.Create()
.WithTraceId(traceId)
.Build();
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithAPIKey("123456"));
dataSender.WithHttpHandlerImpl((serializedJson) =>
{
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK);
return(Task.FromResult(response));
});
var response = dataSender.SendDataAsync(spanBatch).Result;
Assert.AreEqual(NewRelicResponseStatus.DidNotSend_NoData, response.ResponseStatus);
}
public void AddVersionInfo(string productName, string productVersion)
{
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithApiKey("123456"));
var fieldInfo = dataSender.GetType().GetField("_userAgent", BindingFlags.NonPublic | BindingFlags.Instance);
var userAgentValueBefore = fieldInfo.GetValue(dataSender);
var expectedUserAgentValue = userAgentValueBefore?.ToString();
if (!string.IsNullOrEmpty(productName) && !string.IsNullOrEmpty(productVersion))
{
expectedUserAgentValue = userAgentValueBefore + " " + $@"{productName}/{productVersion}";
}
dataSender.AddVersionInfo(productName, productVersion);
var userAgentValueAfter = fieldInfo.GetValue(dataSender);
Assert.AreEqual(expectedUserAgentValue, userAgentValueAfter);
}
public void DateSpecificRetry_CorrectDelayDuration()
{
var traceId = "123";
var retryDuration = TimeSpan.FromSeconds(10);
var retryDurationMs = retryDuration.TotalMilliseconds;
var errorMargin = TimeSpan.FromMilliseconds(50).TotalMilliseconds;
var spanBatch = SpanBatchBuilder.Create()
.WithTraceId(traceId)
.WithSpan(SpanBuilder.Create("TestSpan").Build())
.Build();
var config = new TelemetryConfiguration().WithAPIKey("12345");
var dataSender = new SpanDataSender(config);
//Mock out the communication layer
dataSender.WithHttpHandlerImpl((serializedJson) =>
{
var response = new HttpResponseMessage((System.Net.HttpStatusCode) 429);
var retryOnSpecificTime = DateTimeOffset.Now + retryDuration;
response.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(retryOnSpecificTime);
return(Task.FromResult(response));
});
var capturedDelays = new List <int>();
dataSender.WithDelayFunction((delay) =>
{
capturedDelays.Add(delay);
return(Task.Delay(0));
});
var response = dataSender.SendDataAsync(spanBatch).Result;
Assert.AreEqual(NewRelicResponseStatus.Failure, response.ResponseStatus);
Assert.AreEqual(config.MaxRetryAttempts, capturedDelays.Count);
Assert.AreEqual(System.Net.HttpStatusCode.RequestTimeout, response.HttpStatusCode);
Assert.IsTrue(capturedDelays.All(x => x >= retryDurationMs - errorMargin && x <= retryDurationMs + errorMargin),
"Expected duration out of range");
}
public WeatherForecastController()
{
// Obtain the API from the Web.Config file
var apiKey = ConfigurationManager.AppSettings["NewRelic.Telemetry.ApiKey"];
// Create a Logger Factory using settings in the Web.Config File
var loggerConfig = new LoggerConfiguration()
.ReadFrom.AppSettings();
var loggerFactory = new LoggerFactory()
.AddSerilog(loggerConfig.CreateLogger());
_logger = loggerFactory.CreateLogger <WeatherForecastController>();
// Create a new Telemetry Configuration Object with the API Key retrieved
// from the Web.Config
var telemetryConfig = new TelemetryConfiguration().WithAPIKey(apiKey);
// Instantiate the SpanDataSender which manages the communication with New
// Relic endpoints
_spanDataSender = new SpanDataSender(telemetryConfig, loggerFactory);
}
/// <summary>
/// Disables Tracing
/// </summary>
public static void DisableTracing()
{
_dataSender = null;
}
/// <summary>
/// Starts Tracing
/// </summary>
public static void EnableTracing()
{
_dataSender = new SpanDataSender(_currentConfig, _loggerFactory);
}
/// <summary>
/// Use dependency injection in the constructor to pass in the Logger Factory and
/// the Configuration Provider.
/// </summary>
public WeatherForecastController(ILoggerFactory loggerFactory, SpanDataSender spanDataSender)
{
_spanDataSender = spanDataSender;
_logger = loggerFactory.CreateLogger <WeatherForecastController>();
}
async public Task RequestTooLarge_SplitSuccess()
{
const int expectedCountSpans = 9;
const int expectedCountCallsSendData = 7;
const int expectedCountSuccessfulSpanBatches = 4;
const int expectedCountDistinctTraceIds = 1;
const int expectedCountSpanBatchAttribSets = 1;
const string expectedTraceID = "TestTrace";
var actualCountCallsSendData = 0;
// Arrange
var successfulSpanBatches = new List <SpanBatch>();
var dataSender = new SpanDataSender(new TelemetryConfiguration().WithApiKey("123456"));
var okJsons = new List <string>();
// Mock the behavior to return EntityTooLarge for any span batch with 4 or more spans.
// Anything with less than 4 will return success.
dataSender.WithCaptureSendDataAsyncDelegate((spanBatch, retryNum) =>
{
actualCountCallsSendData++;
if (spanBatch.Spans.Count < 4)
{
okJsons.Add(spanBatch.ToJson());
successfulSpanBatches.Add(spanBatch);
}
});
dataSender.WithHttpHandlerImpl((json) =>
{
var response = okJsons.Contains(json)
? new HttpResponseMessage(System.Net.HttpStatusCode.OK)
: new HttpResponseMessage(System.Net.HttpStatusCode.RequestEntityTooLarge);
return(Task.FromResult(response));
});
var attribs = new Dictionary <string, object>()
{
{ "testAttrib1", "testAttribValue1" }
};
var spanBatchBuilder = SpanBatchBuilder.Create()
.WithTraceId(expectedTraceID)
.WithAttributes(attribs);
for (var i = 0; i < expectedCountSpans; i++)
{
spanBatchBuilder.WithSpan(SpanBuilder.Create(i.ToString()).Build());
}
var spanBatch = spanBatchBuilder.Build();
// Act
await dataSender.SendDataAsync(spanBatch);
// Assert
Assert.AreEqual(expectedCountCallsSendData, actualCountCallsSendData);
//Test the Spans
Assert.AreEqual(expectedCountSuccessfulSpanBatches, successfulSpanBatches.Count, "Unexpected number of calls");
Assert.AreEqual(expectedCountSpans, successfulSpanBatches.SelectMany(x => x.Spans).Count(), "Unexpected number of successful Spans");
Assert.AreEqual(expectedCountSpans, successfulSpanBatches.SelectMany(x => x.Spans).Select(x => x.Id).Distinct().Count(), "All Spans should be unique (spanId)");
//Test the attributes on the spanbatch
Assert.AreEqual(expectedCountDistinctTraceIds, successfulSpanBatches.Select(x => x.CommonProperties.TraceId).Distinct().Count(), "The traceId on split batches are not the same");
Assert.AreEqual(expectedTraceID, successfulSpanBatches.FirstOrDefault().CommonProperties.TraceId, "The traceId on split batches does not match the original traceId");
Assert.AreEqual(expectedCountSpanBatchAttribSets, successfulSpanBatches.Select(x => x.CommonProperties.Attributes).Distinct().Count(), "The attributes on all span batches should be the same");
Assert.AreEqual(attribs, successfulSpanBatches.Select(x => x.CommonProperties.Attributes).FirstOrDefault(), "The Span Batch attribute values on split batches do not match the attributes of the original span batch.");
}
