public async Task <TelemetryPushResult> PushTelemetry(TelemetryData data) { try { // have to buffer in memory so we know the content length var serializedData = SerializeTelemetry(data); var bytes = Encoding.UTF8.GetBytes(serializedData); var request = _requestFactory.Create(_endpoint); request.AddHeader(TelemetryConstants.ApiVersionHeader, data.ApiVersion); request.AddHeader(TelemetryConstants.RequestTypeHeader, data.RequestType); using var response = await request.PostAsync(new ArraySegment <byte>(bytes), "application/json").ConfigureAwait(false); if (response.StatusCode is >= 200 and < 300) { Log.Debug("Telemetry sent successfully"); return(TelemetryPushResult.Success); } else if (response.StatusCode == 404) { Log.Debug("Error sending telemetry: 404. Disabling further telemetry, as endpoint '{Endpoint}' not found", _requestFactory.Info(_endpoint)); return(TelemetryPushResult.FatalError); } else { Log.Debug <string, int>("Error sending telemetry to '{Endpoint}' {StatusCode} ", _requestFactory.Info(_endpoint), response.StatusCode); return(TelemetryPushResult.TransientFailure); } }
public IApiRequest Create(Uri endpoint) { var request = _realFactory.Create(endpoint); return(new FakeApiRequest(request)); }
public async Task <bool> SendLogsAsync(ArraySegment <byte> logs, int numberOfLogs) { var retriesRemaining = MaxNumberRetries - 1; var nextSleepDuration = InitialSleepDurationMs; Log.Debug <int>("Sending {Count} logs to the logs intake", numberOfLogs); while (true) { IApiRequest request; try { request = _apiRequestFactory.Create(_logsIntakeEndpoint); } catch (Exception ex) { Log.Error(ex, "An error occurred while generating request to send logs to the intake at {IntakeEndpoint}", _apiRequestFactory.Info(_logsIntakeEndpoint)); return(false); } // Set additional headers request.AddHeader(IntakeHeaderNameApiKey, _apiKey); Exception?exception = null; var isFinalTry = retriesRemaining <= 0; var shouldRetry = true; try { IApiResponse?response = null; try { // TODO: Metrics/Telemetry? response = await request.PostAsync(logs, MimeType).ConfigureAwait(false); if (response.StatusCode is >= 200 and < 300) { Log.Debug <int>("Successfully sent {Count} logs to the intake", numberOfLogs); return(true); } shouldRetry = response.StatusCode switch { 400 => false, // Bad request (likely an issue in the payload formatting) 401 => false, // Unauthorized (likely a missing API Key) 403 => false, // Permission issue (likely using an invalid API Key) 408 => true, // Request Timeout, request should be retried after some time 413 => false, // Payload too large (batch is above 5MB uncompressed) 429 => true, // Too Many Requests, request should be retried after some time >= 400 and < 500 => false, // generic "client" error, don't retry _ => true // Something else, probably server error, do retry }; if (!shouldRetry || isFinalTry) { try { var responseContent = await response.ReadAsStringAsync().ConfigureAwait(false); Log.Error <int, string>("Failed to submit logs with status code {StatusCode} and message: {ResponseContent}", response.StatusCode, responseContent); } catch (Exception ex) { Log.Error <int>(ex, "Unable to read response for failed request with status code {StatusCode}", response.StatusCode); } } } finally { response?.Dispose(); } } catch (Exception ex) { exception = ex; #if DEBUG if (ex.InnerException is InvalidOperationException) { Log.Error <int, string>(ex, "An error occurred while sending {Count} logs to the intake at {IntakeEndpoint}", numberOfLogs, _apiRequestFactory.Info(_logsIntakeEndpoint)); return(false); } #endif } // Error handling block if (!shouldRetry || isFinalTry) { // stop retrying Log.Error <int, string>(exception, "An error occurred while sending {Count} traces to the intake at {IntakeEndpoint}", numberOfLogs, _apiRequestFactory.Info(_logsIntakeEndpoint)); return(false); } // Before retry delay if (IsSocketException(exception)) { Log.Debug(exception, "Unable to communicate with the logs intake at {IntakeEndpoint}", _apiRequestFactory.Info(_logsIntakeEndpoint)); } // Execute retry delay await Task.Delay(nextSleepDuration).ConfigureAwait(false); retriesRemaining--; nextSleepDuration *= 2; } }
public async Task SendPayloadAsync(EventsPayload payload) { var numberOfTraces = payload.Count; var tracesEndpoint = payload.Url; // retry up to 5 times with exponential back-off const int retryLimit = 5; var retryCount = 1; var sleepDuration = 100; // in milliseconds var payloadMimeType = MimeTypes.MsgPack; var payloadBytes = payload.ToArray(); Log.Information($"Sending ({numberOfTraces} events) {payloadBytes.Length.ToString("N0")} bytes..."); while (true) { IApiRequest request; try { request = _apiRequestFactory.Create(tracesEndpoint); request.AddHeader(ApiKeyHeader, CIVisibility.Settings.ApiKey); } catch (Exception ex) { Log.Error(ex, "An error occurred while generating http request to send events to {AgentEndpoint}", _apiRequestFactory.Info(tracesEndpoint)); return; } bool success = false; Exception exception = null; bool isFinalTry = retryCount >= retryLimit; try { success = await SendPayloadAsync(new ArraySegment <byte>(payloadBytes), payloadMimeType, request, isFinalTry).ConfigureAwait(false); } catch (Exception ex) { exception = ex; if (_globalSettings.DebugEnabled) { if (ex.InnerException is InvalidOperationException ioe) { Log.Error <int, string>(ex, "An error occurred while sending {Count} events to {AgentEndpoint}", numberOfTraces, _apiRequestFactory.Info(tracesEndpoint)); return; } } } // Error handling block if (!success) { if (isFinalTry) { // stop retrying Log.Error <int, string>(exception, "An error occurred while sending {Count} events to {AgentEndpoint}", numberOfTraces, _apiRequestFactory.Info(tracesEndpoint)); return; } // Before retry delay bool isSocketException = false; Exception innerException = exception; while (innerException != null) { if (innerException is SocketException) { isSocketException = true; break; } innerException = innerException.InnerException; } if (isSocketException) { Log.Debug(exception, "Unable to communicate with {AgentEndpoint}", _apiRequestFactory.Info(tracesEndpoint)); } // Execute retry delay await Task.Delay(sleepDuration).ConfigureAwait(false); retryCount++; sleepDuration *= 2; continue; } Log.Debug <int, string>("Successfully sent {Count} events to {AgentEndpoint}", numberOfTraces, _apiRequestFactory.Info(tracesEndpoint)); return; } }