public async Task <TelemetryPushResult> PushTelemetry(TelemetryData data)
{
try
{
// have to buffer in memory so we know the content length
var serializedData = SerializeTelemetry(data);
var bytes = Encoding.UTF8.GetBytes(serializedData);
var request = _requestFactory.Create(_endpoint);
request.AddHeader(TelemetryConstants.ApiVersionHeader, data.ApiVersion);
request.AddHeader(TelemetryConstants.RequestTypeHeader, data.RequestType);
using var response = await request.PostAsync(new ArraySegment <byte>(bytes), "application/json").ConfigureAwait(false);
if (response.StatusCode is >= 200 and < 300)
{
Log.Debug("Telemetry sent successfully");
return(TelemetryPushResult.Success);
}
else if (response.StatusCode == 404)
{
Log.Debug("Error sending telemetry: 404. Disabling further telemetry, as endpoint '{Endpoint}' not found", _requestFactory.Info(_endpoint));
return(TelemetryPushResult.FatalError);
}
else
{
Log.Debug <string, int>("Error sending telemetry to '{Endpoint}' {StatusCode} ", _requestFactory.Info(_endpoint), response.StatusCode);
return(TelemetryPushResult.TransientFailure);
}
}
public IApiRequest Create(Uri endpoint)
{
var request = _realFactory.Create(endpoint);
return(new FakeApiRequest(request));
}
public async Task <bool> SendLogsAsync(ArraySegment <byte> logs, int numberOfLogs)
{
var retriesRemaining = MaxNumberRetries - 1;
var nextSleepDuration = InitialSleepDurationMs;
Log.Debug <int>("Sending {Count} logs to the logs intake", numberOfLogs);
while (true)
{
IApiRequest request;
try
{
request = _apiRequestFactory.Create(_logsIntakeEndpoint);
}
catch (Exception ex)
{
Log.Error(ex, "An error occurred while generating request to send logs to the intake at {IntakeEndpoint}", _apiRequestFactory.Info(_logsIntakeEndpoint));
return(false);
}
// Set additional headers
request.AddHeader(IntakeHeaderNameApiKey, _apiKey);
Exception?exception = null;
var isFinalTry = retriesRemaining <= 0;
var shouldRetry = true;
try
{
IApiResponse?response = null;
try
{
// TODO: Metrics/Telemetry?
response = await request.PostAsync(logs, MimeType).ConfigureAwait(false);
if (response.StatusCode is >= 200 and < 300)
{
Log.Debug <int>("Successfully sent {Count} logs to the intake", numberOfLogs);
return(true);
}
shouldRetry = response.StatusCode switch
{
400 => false, // Bad request (likely an issue in the payload formatting)
401 => false, // Unauthorized (likely a missing API Key)
403 => false, // Permission issue (likely using an invalid API Key)
408 => true, // Request Timeout, request should be retried after some time
413 => false, // Payload too large (batch is above 5MB uncompressed)
429 => true, // Too Many Requests, request should be retried after some time
>= 400 and < 500 => false, // generic "client" error, don't retry
_ => true // Something else, probably server error, do retry
};
if (!shouldRetry || isFinalTry)
{
try
{
var responseContent = await response.ReadAsStringAsync().ConfigureAwait(false);
Log.Error <int, string>("Failed to submit logs with status code {StatusCode} and message: {ResponseContent}", response.StatusCode, responseContent);
}
catch (Exception ex)
{
Log.Error <int>(ex, "Unable to read response for failed request with status code {StatusCode}", response.StatusCode);
}
}
}
finally
{
response?.Dispose();
}
}
catch (Exception ex)
{
exception = ex;
#if DEBUG
if (ex.InnerException is InvalidOperationException)
{
Log.Error <int, string>(ex, "An error occurred while sending {Count} logs to the intake at {IntakeEndpoint}", numberOfLogs, _apiRequestFactory.Info(_logsIntakeEndpoint));
return(false);
}
#endif
}
// Error handling block
if (!shouldRetry || isFinalTry)
{
// stop retrying
Log.Error <int, string>(exception, "An error occurred while sending {Count} traces to the intake at {IntakeEndpoint}", numberOfLogs, _apiRequestFactory.Info(_logsIntakeEndpoint));
return(false);
}
// Before retry delay
if (IsSocketException(exception))
{
Log.Debug(exception, "Unable to communicate with the logs intake at {IntakeEndpoint}", _apiRequestFactory.Info(_logsIntakeEndpoint));
}
// Execute retry delay
await Task.Delay(nextSleepDuration).ConfigureAwait(false);
retriesRemaining--;
nextSleepDuration *= 2;
}
}
public async Task SendPayloadAsync(EventsPayload payload)
{
var numberOfTraces = payload.Count;
var tracesEndpoint = payload.Url;
// retry up to 5 times with exponential back-off
const int retryLimit = 5;
var retryCount = 1;
var sleepDuration = 100; // in milliseconds
var payloadMimeType = MimeTypes.MsgPack;
var payloadBytes = payload.ToArray();
Log.Information($"Sending ({numberOfTraces} events) {payloadBytes.Length.ToString("N0")} bytes...");
while (true)
{
IApiRequest request;
try
{
request = _apiRequestFactory.Create(tracesEndpoint);
request.AddHeader(ApiKeyHeader, CIVisibility.Settings.ApiKey);
}
catch (Exception ex)
{
Log.Error(ex, "An error occurred while generating http request to send events to {AgentEndpoint}", _apiRequestFactory.Info(tracesEndpoint));
return;
}
bool success = false;
Exception exception = null;
bool isFinalTry = retryCount >= retryLimit;
try
{
success = await SendPayloadAsync(new ArraySegment <byte>(payloadBytes), payloadMimeType, request, isFinalTry).ConfigureAwait(false);
}
catch (Exception ex)
{
exception = ex;
if (_globalSettings.DebugEnabled)
{
if (ex.InnerException is InvalidOperationException ioe)
{
Log.Error <int, string>(ex, "An error occurred while sending {Count} events to {AgentEndpoint}", numberOfTraces, _apiRequestFactory.Info(tracesEndpoint));
return;
}
}
}
// Error handling block
if (!success)
{
if (isFinalTry)
{
// stop retrying
Log.Error <int, string>(exception, "An error occurred while sending {Count} events to {AgentEndpoint}", numberOfTraces, _apiRequestFactory.Info(tracesEndpoint));
return;
}
// Before retry delay
bool isSocketException = false;
Exception innerException = exception;
while (innerException != null)
{
if (innerException is SocketException)
{
isSocketException = true;
break;
}
innerException = innerException.InnerException;
}
if (isSocketException)
{
Log.Debug(exception, "Unable to communicate with {AgentEndpoint}", _apiRequestFactory.Info(tracesEndpoint));
}
// Execute retry delay
await Task.Delay(sleepDuration).ConfigureAwait(false);
retryCount++;
sleepDuration *= 2;
continue;
}
Log.Debug <int, string>("Successfully sent {Count} events to {AgentEndpoint}", numberOfTraces, _apiRequestFactory.Info(tracesEndpoint));
return;
}
}