private void HandleTransmissionSentEvent(object sender, TransmissionProcessedEventArgs e) { HttpWebResponseWrapper httpWebResponseWrapper = e.Response; if (httpWebResponseWrapper != null) { AdditionalVerboseTracing(httpWebResponseWrapper.Content); if (httpWebResponseWrapper.StatusCode == ResponseStatusCodes.Success || httpWebResponseWrapper.StatusCode == ResponseStatusCodes.PartialSuccess) { // There is no further action for ErrorHandlingTransmissionPolicy here as transmission is success/partial success. return; } switch (httpWebResponseWrapper.StatusCode) { case ResponseStatusCodes.BadGateway: case ResponseStatusCodes.GatewayTimeout: case ResponseStatusCodes.RequestTimeout: case ResponseStatusCodes.ServiceUnavailable: case ResponseStatusCodes.InternalServerError: case ResponseStatusCodes.UnknownNetworkError: // Disable sending and buffer capacity (Enqueue will enqueue to the Storage) this.MaxSenderCapacity = 0; this.MaxBufferCapacity = 0; this.LogCapacityChanged(); this.Apply(); this.backoffLogicManager.ReportBackoffEnabled((int)httpWebResponseWrapper.StatusCode); this.Transmitter.Enqueue(e.Transmission); this.pauseTimer.Delay = this.backoffLogicManager.GetBackOffTimeInterval(httpWebResponseWrapper.RetryAfterHeader); this.pauseTimer.Start( () => { this.MaxBufferCapacity = null; this.MaxSenderCapacity = null; this.LogCapacityChanged(); this.Apply(); this.backoffLogicManager.ReportBackoffDisabled(); return(Task.FromResult <object>(null)); }); break; default: // We are losing data here but that is intentional as the response code is // not in the whitelisted set to attempt retry. TelemetryChannelEventSource.Log.TransmissionDataNotRetriedForNonWhitelistedResponse(e.Transmission.Id, httpWebResponseWrapper.StatusCode.ToString(CultureInfo.InvariantCulture)); break; } } else { // Data loss Unknown Exception // We are losing data here (we did not upload failed transaction back). // We got unknown exception. if (e.Exception != null) { TelemetryChannelEventSource.Log.TransmissionDataLossError(e.Transmission.Id, e.Exception.Message); } else { TelemetryChannelEventSource.Log.TransmissionDataLossError(e.Transmission.Id, "Unknown Exception Message"); } } }
private async Task StartSending(Transmission transmission) { SdkInternalOperationsMonitor.Enter(); try { Exception exception = null; HttpWebResponseWrapper responseContent = null; // Locally self-throttle this payload before we send it Transmission acceptedTransmission = this.Throttle(transmission); // Now that we've self-imposed a throttle, we can try to send the remaining data try { TelemetryChannelEventSource.Log.TransmissionSendStarted(acceptedTransmission.Id); responseContent = await acceptedTransmission.SendAsync().ConfigureAwait(false); } catch (Exception e) { exception = e; } finally { int currentCapacity = Interlocked.Decrement(ref this.transmissionCount); if (exception != null) { TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning(acceptedTransmission.Id, exception.ToString()); } else { if (responseContent != null) { if (responseContent.StatusCode < 400) { TelemetryChannelEventSource.Log.TransmissionSentSuccessfully(acceptedTransmission.Id, currentCapacity); } else { TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning( acceptedTransmission.Id, responseContent.StatusCode.ToString(CultureInfo.InvariantCulture)); } if (TelemetryChannelEventSource.IsVerboseEnabled) { TelemetryChannelEventSource.Log.RawResponseFromAIBackend(acceptedTransmission.Id, responseContent.Content); } } } if (responseContent == null && exception is HttpRequestException) { // HttpClient.SendAsync throws HttpRequestException on the following scenarios: // "The request failed due to an underlying issue such as network connectivity, DNS failure, server certificate validation or timeout." // https://docs.microsoft.com/en-us/dotnet/api/system.net.http.httpclient.sendasync?view=netstandard-1.6 responseContent = new HttpWebResponseWrapper() { // Expectation is that RetryPolicy will attempt retry for this status. StatusCode = ResponseStatusCodes.UnknownNetworkError }; } this.OnTransmissionSent(new TransmissionProcessedEventArgs(acceptedTransmission, exception, responseContent)); } } finally { SdkInternalOperationsMonitor.Exit(); } }
private void HandleTransmissionSentEvent(object sender, TransmissionProcessedEventArgs e) { var webException = e.Exception as WebException; if (webException != null) { HttpWebResponseWrapper httpWebResponse = e.Response; if (httpWebResponse != null) { TelemetryChannelEventSource.Log.TransmissionSendingFailedWebExceptionWarning( e.Transmission.Id, webException.Message, (int)httpWebResponse.StatusCode, httpWebResponse.StatusDescription); this.AdditionalVerboseTracing((HttpWebResponse)webException.Response); switch (httpWebResponse.StatusCode) { case ResponseStatusCodes.RequestTimeout: case ResponseStatusCodes.ServiceUnavailable: case ResponseStatusCodes.InternalServerError: // Disable sending and buffer capacity (Enqueue will enqueue to the Storage) this.MaxSenderCapacity = 0; this.MaxBufferCapacity = 0; this.LogCapacityChanged(); this.Apply(); this.backoffLogicManager.ReportBackoffEnabled((int)httpWebResponse.StatusCode); this.Transmitter.Enqueue(e.Transmission); this.pauseTimer.Delay = this.backoffLogicManager.GetBackOffTimeInterval(httpWebResponse.RetryAfterHeader); this.pauseTimer.Start( () => { this.MaxBufferCapacity = null; this.MaxSenderCapacity = null; this.LogCapacityChanged(); this.Apply(); this.backoffLogicManager.ReportBackoffDisabled(); return(TaskEx.FromResult <object>(null)); }); break; } } else { // We are loosing data here (we did not upload failed transaction back). // We did not get response back. TelemetryChannelEventSource.Log.TransmissionSendingFailedWebExceptionWarning(e.Transmission.Id, webException.Message, (int)HttpStatusCode.InternalServerError, null); } } else { if (e.Exception != null) { // We are loosing data here (we did not upload failed transaction back). // We got unknown exception. TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning(e.Transmission.Id, e.Exception.Message); } } }
public TransmissionProcessedEventArgs(Transmission transmission, Exception exception = null, HttpWebResponseWrapper response = null) { // Fix missing arguments if not passed in if (exception != null && response == null && exception is WebException && ((WebException)exception).Response is HttpWebResponse) { HttpWebResponse exceptionResponse = (HttpWebResponse)((WebException)exception).Response; response = new HttpWebResponseWrapper() { StatusCode = (int)exceptionResponse.StatusCode, StatusDescription = exceptionResponse.StatusDescription, RetryAfterHeader = exceptionResponse.Headers?.Get("Retry-After") }; } this.Transmission = transmission; this.Exception = exception; this.Response = response; }
public TransmissionProcessedEventArgs(Transmission transmission, Exception exception = null, HttpWebResponseWrapper response = null) { this.Transmission = transmission; this.Exception = exception; this.Response = response; }
private static bool IsAccessTokenExpired(WebResponse response) { if (response == null) return false; using (IHttpWebResponseWrapper responseWrapper = new HttpWebResponseWrapper(response)) { JObject json = JObject.Parse(responseWrapper.Content); { return ((string)json["Error"]).Equals("invalid_grant"); } } }
/// <summary> /// Executes the request that the current transmission represents. /// </summary> /// <returns>The task to await.</returns> public virtual async Task <HttpWebResponseWrapper> SendAsync() { if (Interlocked.CompareExchange(ref this.isSending, 1, 0) != 0) { throw new InvalidOperationException("SendAsync is already in progress."); } try { using (MemoryStream contentStream = new MemoryStream(this.Content)) { HttpRequestMessage request = this.CreateRequestMessage(this.EndpointAddress, contentStream); HttpWebResponseWrapper wrapper = null; long responseDurationInMs = 0; try { using (var ct = new CancellationTokenSource(this.Timeout)) { // HttpClient.SendAsync throws HttpRequestException only on the following scenarios: // "The request failed due to an underlying issue such as network connectivity, DNS failure, server certificate validation or timeout." // i.e for Server errors (500 status code), no exception is thrown. Hence this method should read the response and status code, // and return correct HttpWebResponseWrapper to give any Retry policies a chance to retry as needed. var stopwatch = new Stopwatch(); stopwatch.Start(); using (var response = await client.SendAsync(request, ct.Token).ConfigureAwait(false)) { stopwatch.Stop(); responseDurationInMs = stopwatch.ElapsedMilliseconds; CoreEventSource.Log.IngestionResponseTime(response != null ? (int)response.StatusCode : -1, responseDurationInMs); // Log ingestion respose time as event counter metric. CoreEventSource.Log.IngestionResponseTimeEventCounter(stopwatch.ElapsedMilliseconds); if (response != null) { wrapper = new HttpWebResponseWrapper { StatusCode = (int)response.StatusCode, StatusDescription = response.ReasonPhrase, // maybe not required? }; wrapper.RetryAfterHeader = response.Headers?.RetryAfter?.ToString(); if (response.StatusCode == HttpStatusCode.PartialContent) { if (response.Content != null) { // Read the entire response body only on PartialContent for perf reasons. // This cannot be avoided as response tells which items are to be resubmitted. wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } } if (CoreEventSource.IsVerboseEnabled && response.StatusCode != HttpStatusCode.PartialContent) { // Read the entire response body only on VerboseTracing for perf reasons. try { if (response.Content != null) { wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } } catch (Exception) { // Swallow any exception here as this code is for tracing purposes only and should never throw. } } } } } } catch (OperationCanceledException) { wrapper = new HttpWebResponseWrapper { StatusCode = (int)HttpStatusCode.RequestTimeout, }; } finally { try { // Initiates event notification to subscriber with Transmission and TransmissionStatusEventArgs. this.TransmissionStatusEvent?.Invoke(this, new TransmissionStatusEventArgs(wrapper ?? new HttpWebResponseWrapper() { StatusCode = 999 }, responseDurationInMs)); } catch (Exception ex) { CoreEventSource.Log.TransmissionStatusEventFailed(ex); } } return(wrapper); } } finally { Interlocked.Exchange(ref this.isSending, 0); } }
/// <summary> /// Executes the request that the current transmission represents. /// </summary> /// <returns>The task to await.</returns> public virtual async Task <HttpWebResponseWrapper> SendAsync() { if (Interlocked.CompareExchange(ref this.isSending, 1, 0) != 0) { throw new InvalidOperationException("SendAsync is already in progress."); } try { using (MemoryStream contentStream = new MemoryStream(this.Content)) { HttpRequestMessage request = this.CreateRequestMessage(this.EndpointAddress, contentStream); HttpWebResponseWrapper wrapper = null; try { using (var ct = new CancellationTokenSource(this.Timeout)) { // HttpClient.SendAsync throws HttpRequestException only on the following scenarios: // "The request failed due to an underlying issue such as network connectivity, DNS failure, server certificate validation or timeout." // i.e for Server errors (500 status code), no exception is thrown. Hence this method should read the response and status code, // and return correct HttpWebResponseWrapper to give any Retry policies a chance to retry as needed. using (var response = await client.SendAsync(request, ct.Token).ConfigureAwait(false)) { if (response != null) { wrapper = new HttpWebResponseWrapper { StatusCode = (int)response.StatusCode, StatusDescription = response.ReasonPhrase // maybe not required? }; wrapper.RetryAfterHeader = response.Headers?.RetryAfter?.ToString(); if (response.StatusCode == HttpStatusCode.PartialContent) { if (response.Content != null) { // Read the entire response body only on PartialContent for perf reasons. // This cannot be avoided as response tells which items are to be resubmitted. wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } } if (CoreEventSource.IsVerboseEnabled && response.StatusCode != HttpStatusCode.PartialContent) { // Read the entire response body only on VerboseTracing for perf reasons. try { if (response.Content != null) { wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } } catch (Exception) { // Swallow any exception here as this code is for tracing purposes only and should never throw. } } } } } } catch (TaskCanceledException) { wrapper = new HttpWebResponseWrapper { StatusCode = (int)HttpStatusCode.RequestTimeout }; } return(wrapper); } } finally { Interlocked.Exchange(ref this.isSending, 0); } }
internal APIResponse GetResponse(HttpWebResponseWrapper response) { APIResponse apiResponse = new APIResponse(); //Extract content using (Stream stream = response.GetResponseStream()) { StreamReader sr = new StreamReader(stream); apiResponse.Content = (new JsonTranslator()).Decode(sr.ReadToEnd()); sr.Close(); } apiResponse.Header = response.Headers; apiResponse.StatusCode = (int)response.StatusCode; if (response.Headers != null) { bool hasMore; if (response.Headers.Get("x-hasmore") != null && Boolean.TryParse(response.Headers.GetValues("x-hasmore").First().ToString(), out hasMore)) apiResponse.HasMore = hasMore; int apiLimit = 0; if (response.Headers.Get("x-apilimit-remaining") != null && int.TryParse(response.Headers.GetValues("x-apilimit-remaining").First().Split('/')[0], out apiLimit)) apiResponse.ApiLimit = apiLimit; int apiLimitRemaining = 0; if (response.Headers.Get("x-apilimit-remaining") != null && int.TryParse(response.Headers.GetValues("x-apilimit-remaining").First().Split('/')[1], out apiLimitRemaining)) apiResponse.ApiLimitRemaining = apiLimitRemaining; } //Recover object if error if (apiResponse.Content is JObject && apiResponse.Content.responseCode != null) { apiResponse.StatusCode = apiResponse.Content.responseCode; apiResponse.Error.Message = apiResponse.Content.message; if (apiResponse.Content.errors != null) { foreach (var itemError in apiResponse.Content.errors) { apiResponse.Error.ErrorDetails.Add( new ErrorDetail { Message = itemError.message, Code = itemError.code, MoreInfo = itemError.moreInfo } ); } } } return apiResponse; }
private async Task StartSending(Transmission transmission) { SdkInternalOperationsMonitor.Enter(); try { Exception exception = null; HttpWebResponseWrapper responseContent = null; // Locally self-throttle this payload before we send it Transmission acceptedTransmission = this.Throttle(transmission); // Now that we've self-imposed a throttle, we can try to send the remaining data try { TelemetryChannelEventSource.Log.TransmissionSendStarted(acceptedTransmission.Id); responseContent = await acceptedTransmission.SendAsync().ConfigureAwait(false); } catch (Exception e) { exception = e; } finally { int currentCapacity = Interlocked.Decrement(ref this.transmissionCount); if (exception == null) { TelemetryChannelEventSource.Log.TransmissionSentSuccessfully(acceptedTransmission.Id, currentCapacity); } else { TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning(acceptedTransmission.Id, exception.ToString()); } if (responseContent == null && exception is WebException) { HttpWebResponse response = (HttpWebResponse)((WebException)exception).Response; if (response != null) { responseContent = new HttpWebResponseWrapper() { StatusCode = (int)response.StatusCode, StatusDescription = response.StatusDescription, RetryAfterHeader = response.Headers?.Get("Retry-After") }; } else { responseContent = new HttpWebResponseWrapper() { StatusCode = 0, StatusDescription = null, RetryAfterHeader = null }; } } this.OnTransmissionSent(new TransmissionProcessedEventArgs(acceptedTransmission, exception, responseContent)); } } finally { SdkInternalOperationsMonitor.Exit(); } }
public async Task TestTransmissionStatusEventWithEventsFromMultipleIKey() { // ARRANGE // Raw response from backend for partial response var ingestionResponse = "{" + "\r\n \"itemsReceived\": 5,\r\n \"itemsAccepted\": 2,\r\n " + "\"errors\": [\r\n {\r\n " + "\"index\": 0,\r\n \"statusCode\": 400,\r\n \"message\": \"Error 1\"\r\n },\r\n {\r\n " + "\"index\": 2,\r\n \"statusCode\": 503,\r\n \"message\": \"Error 2\"\r\n },\r\n {\r\n " + "\"index\": 3,\r\n \"statusCode\": 500,\r\n \"message\": \"Error 3\"\r\n }\r\n ]\r\n}"; // Fake HttpClient will respond back with partial content var handler = new HandlerForFakeHttpClient { InnerHandler = new HttpClientHandler(), OnSendAsync = (req, cancellationToken) => { return(Task.FromResult <HttpResponseMessage>(new HttpResponseMessage { StatusCode = HttpStatusCode.PartialContent, Content = new StringContent(ingestionResponse) })); } }; using (var fakeHttpClient = new HttpClient(handler)) { // Create a list of telemetry which could send information to different instrumentation keys var telemetryItems = new List <ITelemetry>(); EventTelemetry eventTelemetry1 = new EventTelemetry("Event1"); eventTelemetry1.Context.InstrumentationKey = "IKEY_1"; telemetryItems.Add(eventTelemetry1); EventTelemetry eventTelemetry2 = new EventTelemetry("Event2"); eventTelemetry2.Context.InstrumentationKey = "IKEY_2"; telemetryItems.Add(eventTelemetry2); EventTelemetry eventTelemetry3 = new EventTelemetry("Event3"); eventTelemetry3.Context.InstrumentationKey = "IKEY_3"; telemetryItems.Add(eventTelemetry3); EventTelemetry eventTelemetry4 = new EventTelemetry("Event3"); eventTelemetry4.Context.InstrumentationKey = "IKEY_2"; telemetryItems.Add(eventTelemetry4); EventTelemetry eventTelemetry5 = new EventTelemetry("Event5"); eventTelemetry5.Context.InstrumentationKey = "IKEY_1"; telemetryItems.Add(eventTelemetry5); // Serialize the telemetry items before passing to transmission var serializedData = JsonSerializer.Serialize(telemetryItems, true); // Instantiate Transmission with the mock HttpClient Transmission transmission = new Transmission(testUri, serializedData, fakeHttpClient, string.Empty, string.Empty); // VALIDATE transmission.TransmissionStatusEvent += delegate(object sender, TransmissionStatusEventArgs args) { var sendertransmission = sender as Transmission; // convert raw JSON response to Backendresponse object BackendResponse backendResponse = GetBackendResponse(args.Response.Content); // Deserialize telemetry items to identify which items has failed string[] items = JsonSerializer .Deserialize(sendertransmission.Content) .Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); string[] failedItems = new string[3]; int i = 0; // Create a list of failed items foreach (var error in backendResponse.Errors) { failedItems[i++] = items[error.Index]; } Assert.AreEqual((int)HttpStatusCode.PartialContent, args.Response.StatusCode); Assert.AreEqual(5, backendResponse.ItemsReceived); Assert.AreEqual(2, backendResponse.ItemsAccepted); //IKEY_1 int totalItemsForIkey = items.Where(x => x.Contains("IKEY_1")).Count(); int failedItemsForIkey = failedItems.Where(x => x.Contains("IKEY_1")).Count(); Assert.AreEqual(2, totalItemsForIkey); Assert.AreEqual(1, failedItemsForIkey); //IKEY_2 totalItemsForIkey = items.Where(x => x.Contains("IKEY_2")).Count(); failedItemsForIkey = failedItems.Where(x => x.Contains("IKEY_2")).Count(); Assert.AreEqual(2, totalItemsForIkey); Assert.AreEqual(1, failedItemsForIkey); //IKEY_3 totalItemsForIkey = items.Where(x => x.Contains("IKEY_3")).Count(); failedItemsForIkey = failedItems.Where(x => x.Contains("IKEY_3")).Count(); Assert.AreEqual(1, totalItemsForIkey); Assert.AreEqual(1, failedItemsForIkey); }; // ACT HttpWebResponseWrapper result = await transmission.SendAsync(); } }
/// <summary> /// Initializes a new instance of the <see cref="TransmissionStatusEventArgs"/> class. /// </summary> /// <param name="response">Response from ingestion endpoint.</param> /// <param name="responseDurationInMs">Response duration in milliseconds.</param> public TransmissionStatusEventArgs(HttpWebResponseWrapper response, long responseDurationInMs) { this.Response = response; this.ResponseDurationInMs = responseDurationInMs; }
public TransmissionStatusEventArgs(HttpWebResponseWrapper response) : this(response, default) { }
/// <summary> /// Initializes a new instance of the <see cref="TransmissionStatusEventArgs"/> class. /// </summary> /// <param name="response">Response from ingestion endpoint.</param> public TransmissionStatusEventArgs(HttpWebResponseWrapper response) { this.Response = response; }