private void HandleTransmissionSentEvent(object sender, TransmissionProcessedEventArgs e)
{
HttpWebResponseWrapper httpWebResponseWrapper = e.Response;
if (httpWebResponseWrapper != null)
{
AdditionalVerboseTracing(httpWebResponseWrapper.Content);
if (httpWebResponseWrapper.StatusCode == ResponseStatusCodes.Success || httpWebResponseWrapper.StatusCode == ResponseStatusCodes.PartialSuccess)
{
// There is no further action for ErrorHandlingTransmissionPolicy here as transmission is success/partial success.
return;
}
switch (httpWebResponseWrapper.StatusCode)
{
case ResponseStatusCodes.BadGateway:
case ResponseStatusCodes.GatewayTimeout:
case ResponseStatusCodes.RequestTimeout:
case ResponseStatusCodes.ServiceUnavailable:
case ResponseStatusCodes.InternalServerError:
case ResponseStatusCodes.UnknownNetworkError:
// Disable sending and buffer capacity (Enqueue will enqueue to the Storage)
this.MaxSenderCapacity = 0;
this.MaxBufferCapacity = 0;
this.LogCapacityChanged();
this.Apply();
this.backoffLogicManager.ReportBackoffEnabled((int)httpWebResponseWrapper.StatusCode);
this.Transmitter.Enqueue(e.Transmission);
this.pauseTimer.Delay = this.backoffLogicManager.GetBackOffTimeInterval(httpWebResponseWrapper.RetryAfterHeader);
this.pauseTimer.Start(
() =>
{
this.MaxBufferCapacity = null;
this.MaxSenderCapacity = null;
this.LogCapacityChanged();
this.Apply();
this.backoffLogicManager.ReportBackoffDisabled();
return(Task.FromResult <object>(null));
});
break;
default:
// We are losing data here but that is intentional as the response code is
// not in the whitelisted set to attempt retry.
TelemetryChannelEventSource.Log.TransmissionDataNotRetriedForNonWhitelistedResponse(e.Transmission.Id,
httpWebResponseWrapper.StatusCode.ToString(CultureInfo.InvariantCulture));
break;
}
}
else
{
// Data loss Unknown Exception
// We are losing data here (we did not upload failed transaction back).
// We got unknown exception.
if (e.Exception != null)
{
TelemetryChannelEventSource.Log.TransmissionDataLossError(e.Transmission.Id,
e.Exception.Message);
}
else
{
TelemetryChannelEventSource.Log.TransmissionDataLossError(e.Transmission.Id,
"Unknown Exception Message");
}
}
}
private async Task StartSending(Transmission transmission)
{
SdkInternalOperationsMonitor.Enter();
try
{
Exception exception = null;
HttpWebResponseWrapper responseContent = null;
// Locally self-throttle this payload before we send it
Transmission acceptedTransmission = this.Throttle(transmission);
// Now that we've self-imposed a throttle, we can try to send the remaining data
try
{
TelemetryChannelEventSource.Log.TransmissionSendStarted(acceptedTransmission.Id);
responseContent = await acceptedTransmission.SendAsync().ConfigureAwait(false);
}
catch (Exception e)
{
exception = e;
}
finally
{
int currentCapacity = Interlocked.Decrement(ref this.transmissionCount);
if (exception != null)
{
TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning(acceptedTransmission.Id, exception.ToString());
}
else
{
if (responseContent != null)
{
if (responseContent.StatusCode < 400)
{
TelemetryChannelEventSource.Log.TransmissionSentSuccessfully(acceptedTransmission.Id,
currentCapacity);
}
else
{
TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning(
acceptedTransmission.Id,
responseContent.StatusCode.ToString(CultureInfo.InvariantCulture));
}
if (TelemetryChannelEventSource.IsVerboseEnabled)
{
TelemetryChannelEventSource.Log.RawResponseFromAIBackend(acceptedTransmission.Id,
responseContent.Content);
}
}
}
if (responseContent == null && exception is HttpRequestException)
{
// HttpClient.SendAsync throws HttpRequestException on the following scenarios:
// "The request failed due to an underlying issue such as network connectivity, DNS failure, server certificate validation or timeout."
// https://docs.microsoft.com/en-us/dotnet/api/system.net.http.httpclient.sendasync?view=netstandard-1.6
responseContent = new HttpWebResponseWrapper()
{
// Expectation is that RetryPolicy will attempt retry for this status.
StatusCode = ResponseStatusCodes.UnknownNetworkError
};
}
this.OnTransmissionSent(new TransmissionProcessedEventArgs(acceptedTransmission, exception, responseContent));
}
}
finally
{
SdkInternalOperationsMonitor.Exit();
}
}
private void HandleTransmissionSentEvent(object sender, TransmissionProcessedEventArgs e)
{
var webException = e.Exception as WebException;
if (webException != null)
{
HttpWebResponseWrapper httpWebResponse = e.Response;
if (httpWebResponse != null)
{
TelemetryChannelEventSource.Log.TransmissionSendingFailedWebExceptionWarning(
e.Transmission.Id,
webException.Message,
(int)httpWebResponse.StatusCode,
httpWebResponse.StatusDescription);
this.AdditionalVerboseTracing((HttpWebResponse)webException.Response);
switch (httpWebResponse.StatusCode)
{
case ResponseStatusCodes.RequestTimeout:
case ResponseStatusCodes.ServiceUnavailable:
case ResponseStatusCodes.InternalServerError:
// Disable sending and buffer capacity (Enqueue will enqueue to the Storage)
this.MaxSenderCapacity = 0;
this.MaxBufferCapacity = 0;
this.LogCapacityChanged();
this.Apply();
this.backoffLogicManager.ReportBackoffEnabled((int)httpWebResponse.StatusCode);
this.Transmitter.Enqueue(e.Transmission);
this.pauseTimer.Delay = this.backoffLogicManager.GetBackOffTimeInterval(httpWebResponse.RetryAfterHeader);
this.pauseTimer.Start(
() =>
{
this.MaxBufferCapacity = null;
this.MaxSenderCapacity = null;
this.LogCapacityChanged();
this.Apply();
this.backoffLogicManager.ReportBackoffDisabled();
return(TaskEx.FromResult <object>(null));
});
break;
}
}
else
{
// We are loosing data here (we did not upload failed transaction back).
// We did not get response back.
TelemetryChannelEventSource.Log.TransmissionSendingFailedWebExceptionWarning(e.Transmission.Id, webException.Message, (int)HttpStatusCode.InternalServerError, null);
}
}
else
{
if (e.Exception != null)
{
// We are loosing data here (we did not upload failed transaction back).
// We got unknown exception.
TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning(e.Transmission.Id, e.Exception.Message);
}
}
}
public TransmissionProcessedEventArgs(Transmission transmission, Exception exception = null, HttpWebResponseWrapper response = null)
{
// Fix missing arguments if not passed in
if (exception != null && response == null && exception is WebException &&
((WebException)exception).Response is HttpWebResponse)
{
HttpWebResponse exceptionResponse = (HttpWebResponse)((WebException)exception).Response;
response = new HttpWebResponseWrapper()
{
StatusCode = (int)exceptionResponse.StatusCode,
StatusDescription = exceptionResponse.StatusDescription,
RetryAfterHeader = exceptionResponse.Headers?.Get("Retry-After")
};
}
this.Transmission = transmission;
this.Exception = exception;
this.Response = response;
}
public TransmissionProcessedEventArgs(Transmission transmission, Exception exception = null, HttpWebResponseWrapper response = null)
{
this.Transmission = transmission;
this.Exception = exception;
this.Response = response;
}
private static bool IsAccessTokenExpired(WebResponse response)
{
if (response == null)
return false;
using (IHttpWebResponseWrapper responseWrapper = new HttpWebResponseWrapper(response))
{
JObject json = JObject.Parse(responseWrapper.Content);
{
return ((string)json["Error"]).Equals("invalid_grant");
}
}
}
/// <summary>
/// Executes the request that the current transmission represents.
/// </summary>
/// <returns>The task to await.</returns>
public virtual async Task <HttpWebResponseWrapper> SendAsync()
{
if (Interlocked.CompareExchange(ref this.isSending, 1, 0) != 0)
{
throw new InvalidOperationException("SendAsync is already in progress.");
}
try
{
using (MemoryStream contentStream = new MemoryStream(this.Content))
{
HttpRequestMessage request = this.CreateRequestMessage(this.EndpointAddress, contentStream);
HttpWebResponseWrapper wrapper = null;
long responseDurationInMs = 0;
try
{
using (var ct = new CancellationTokenSource(this.Timeout))
{
// HttpClient.SendAsync throws HttpRequestException only on the following scenarios:
// "The request failed due to an underlying issue such as network connectivity, DNS failure, server certificate validation or timeout."
// i.e for Server errors (500 status code), no exception is thrown. Hence this method should read the response and status code,
// and return correct HttpWebResponseWrapper to give any Retry policies a chance to retry as needed.
var stopwatch = new Stopwatch();
stopwatch.Start();
using (var response = await client.SendAsync(request, ct.Token).ConfigureAwait(false))
{
stopwatch.Stop();
responseDurationInMs = stopwatch.ElapsedMilliseconds;
CoreEventSource.Log.IngestionResponseTime(response != null ? (int)response.StatusCode : -1, responseDurationInMs);
// Log ingestion respose time as event counter metric.
CoreEventSource.Log.IngestionResponseTimeEventCounter(stopwatch.ElapsedMilliseconds);
if (response != null)
{
wrapper = new HttpWebResponseWrapper
{
StatusCode = (int)response.StatusCode,
StatusDescription = response.ReasonPhrase, // maybe not required?
};
wrapper.RetryAfterHeader = response.Headers?.RetryAfter?.ToString();
if (response.StatusCode == HttpStatusCode.PartialContent)
{
if (response.Content != null)
{
// Read the entire response body only on PartialContent for perf reasons.
// This cannot be avoided as response tells which items are to be resubmitted.
wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
if (CoreEventSource.IsVerboseEnabled && response.StatusCode != HttpStatusCode.PartialContent)
{
// Read the entire response body only on VerboseTracing for perf reasons.
try
{
if (response.Content != null)
{
wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
catch (Exception)
{
// Swallow any exception here as this code is for tracing purposes only and should never throw.
}
}
}
}
}
}
catch (OperationCanceledException)
{
wrapper = new HttpWebResponseWrapper
{
StatusCode = (int)HttpStatusCode.RequestTimeout,
};
}
finally
{
try
{
// Initiates event notification to subscriber with Transmission and TransmissionStatusEventArgs.
this.TransmissionStatusEvent?.Invoke(this, new TransmissionStatusEventArgs(wrapper ?? new HttpWebResponseWrapper()
{
StatusCode = 999
}, responseDurationInMs));
}
catch (Exception ex)
{
CoreEventSource.Log.TransmissionStatusEventFailed(ex);
}
}
return(wrapper);
}
}
finally
{
Interlocked.Exchange(ref this.isSending, 0);
}
}
/// <summary>
/// Executes the request that the current transmission represents.
/// </summary>
/// <returns>The task to await.</returns>
public virtual async Task <HttpWebResponseWrapper> SendAsync()
{
if (Interlocked.CompareExchange(ref this.isSending, 1, 0) != 0)
{
throw new InvalidOperationException("SendAsync is already in progress.");
}
try
{
using (MemoryStream contentStream = new MemoryStream(this.Content))
{
HttpRequestMessage request = this.CreateRequestMessage(this.EndpointAddress, contentStream);
HttpWebResponseWrapper wrapper = null;
try
{
using (var ct = new CancellationTokenSource(this.Timeout))
{
// HttpClient.SendAsync throws HttpRequestException only on the following scenarios:
// "The request failed due to an underlying issue such as network connectivity, DNS failure, server certificate validation or timeout."
// i.e for Server errors (500 status code), no exception is thrown. Hence this method should read the response and status code,
// and return correct HttpWebResponseWrapper to give any Retry policies a chance to retry as needed.
using (var response = await client.SendAsync(request, ct.Token).ConfigureAwait(false))
{
if (response != null)
{
wrapper = new HttpWebResponseWrapper
{
StatusCode = (int)response.StatusCode,
StatusDescription = response.ReasonPhrase // maybe not required?
};
wrapper.RetryAfterHeader = response.Headers?.RetryAfter?.ToString();
if (response.StatusCode == HttpStatusCode.PartialContent)
{
if (response.Content != null)
{
// Read the entire response body only on PartialContent for perf reasons.
// This cannot be avoided as response tells which items are to be resubmitted.
wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
if (CoreEventSource.IsVerboseEnabled && response.StatusCode != HttpStatusCode.PartialContent)
{
// Read the entire response body only on VerboseTracing for perf reasons.
try
{
if (response.Content != null)
{
wrapper.Content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
catch (Exception)
{
// Swallow any exception here as this code is for tracing purposes only and should never throw.
}
}
}
}
}
}
catch (TaskCanceledException)
{
wrapper = new HttpWebResponseWrapper
{
StatusCode = (int)HttpStatusCode.RequestTimeout
};
}
return(wrapper);
}
}
finally
{
Interlocked.Exchange(ref this.isSending, 0);
}
}
internal APIResponse GetResponse(HttpWebResponseWrapper response)
{
APIResponse apiResponse = new APIResponse();
//Extract content
using (Stream stream = response.GetResponseStream())
{
StreamReader sr = new StreamReader(stream);
apiResponse.Content = (new JsonTranslator()).Decode(sr.ReadToEnd());
sr.Close();
}
apiResponse.Header = response.Headers;
apiResponse.StatusCode = (int)response.StatusCode;
if (response.Headers != null)
{
bool hasMore;
if (response.Headers.Get("x-hasmore") != null && Boolean.TryParse(response.Headers.GetValues("x-hasmore").First().ToString(), out hasMore))
apiResponse.HasMore = hasMore;
int apiLimit = 0;
if (response.Headers.Get("x-apilimit-remaining") != null && int.TryParse(response.Headers.GetValues("x-apilimit-remaining").First().Split('/')[0], out apiLimit))
apiResponse.ApiLimit = apiLimit;
int apiLimitRemaining = 0;
if (response.Headers.Get("x-apilimit-remaining") != null && int.TryParse(response.Headers.GetValues("x-apilimit-remaining").First().Split('/')[1], out apiLimitRemaining))
apiResponse.ApiLimitRemaining = apiLimitRemaining;
}
//Recover object if error
if (apiResponse.Content is JObject && apiResponse.Content.responseCode != null)
{
apiResponse.StatusCode = apiResponse.Content.responseCode;
apiResponse.Error.Message = apiResponse.Content.message;
if (apiResponse.Content.errors != null)
{
foreach (var itemError in apiResponse.Content.errors)
{
apiResponse.Error.ErrorDetails.Add(
new ErrorDetail
{
Message = itemError.message,
Code = itemError.code,
MoreInfo = itemError.moreInfo
}
);
}
}
}
return apiResponse;
}
private async Task StartSending(Transmission transmission)
{
SdkInternalOperationsMonitor.Enter();
try
{
Exception exception = null;
HttpWebResponseWrapper responseContent = null;
// Locally self-throttle this payload before we send it
Transmission acceptedTransmission = this.Throttle(transmission);
// Now that we've self-imposed a throttle, we can try to send the remaining data
try
{
TelemetryChannelEventSource.Log.TransmissionSendStarted(acceptedTransmission.Id);
responseContent = await acceptedTransmission.SendAsync().ConfigureAwait(false);
}
catch (Exception e)
{
exception = e;
}
finally
{
int currentCapacity = Interlocked.Decrement(ref this.transmissionCount);
if (exception == null)
{
TelemetryChannelEventSource.Log.TransmissionSentSuccessfully(acceptedTransmission.Id, currentCapacity);
}
else
{
TelemetryChannelEventSource.Log.TransmissionSendingFailedWarning(acceptedTransmission.Id, exception.ToString());
}
if (responseContent == null && exception is WebException)
{
HttpWebResponse response = (HttpWebResponse)((WebException)exception).Response;
if (response != null)
{
responseContent = new HttpWebResponseWrapper()
{
StatusCode = (int)response.StatusCode,
StatusDescription = response.StatusDescription,
RetryAfterHeader = response.Headers?.Get("Retry-After")
};
}
else
{
responseContent = new HttpWebResponseWrapper()
{
StatusCode = 0,
StatusDescription = null,
RetryAfterHeader = null
};
}
}
this.OnTransmissionSent(new TransmissionProcessedEventArgs(acceptedTransmission, exception, responseContent));
}
}
finally
{
SdkInternalOperationsMonitor.Exit();
}
}
public async Task TestTransmissionStatusEventWithEventsFromMultipleIKey()
{
// ARRANGE
// Raw response from backend for partial response
var ingestionResponse = "{" +
"\r\n \"itemsReceived\": 5,\r\n \"itemsAccepted\": 2,\r\n " +
"\"errors\": [\r\n {\r\n " +
"\"index\": 0,\r\n \"statusCode\": 400,\r\n \"message\": \"Error 1\"\r\n },\r\n {\r\n " +
"\"index\": 2,\r\n \"statusCode\": 503,\r\n \"message\": \"Error 2\"\r\n },\r\n {\r\n " +
"\"index\": 3,\r\n \"statusCode\": 500,\r\n \"message\": \"Error 3\"\r\n }\r\n ]\r\n}";
// Fake HttpClient will respond back with partial content
var handler = new HandlerForFakeHttpClient
{
InnerHandler = new HttpClientHandler(),
OnSendAsync = (req, cancellationToken) =>
{
return(Task.FromResult <HttpResponseMessage>(new HttpResponseMessage {
StatusCode = HttpStatusCode.PartialContent, Content = new StringContent(ingestionResponse)
}));
}
};
using (var fakeHttpClient = new HttpClient(handler))
{
// Create a list of telemetry which could send information to different instrumentation keys
var telemetryItems = new List <ITelemetry>();
EventTelemetry eventTelemetry1 = new EventTelemetry("Event1");
eventTelemetry1.Context.InstrumentationKey = "IKEY_1";
telemetryItems.Add(eventTelemetry1);
EventTelemetry eventTelemetry2 = new EventTelemetry("Event2");
eventTelemetry2.Context.InstrumentationKey = "IKEY_2";
telemetryItems.Add(eventTelemetry2);
EventTelemetry eventTelemetry3 = new EventTelemetry("Event3");
eventTelemetry3.Context.InstrumentationKey = "IKEY_3";
telemetryItems.Add(eventTelemetry3);
EventTelemetry eventTelemetry4 = new EventTelemetry("Event3");
eventTelemetry4.Context.InstrumentationKey = "IKEY_2";
telemetryItems.Add(eventTelemetry4);
EventTelemetry eventTelemetry5 = new EventTelemetry("Event5");
eventTelemetry5.Context.InstrumentationKey = "IKEY_1";
telemetryItems.Add(eventTelemetry5);
// Serialize the telemetry items before passing to transmission
var serializedData = JsonSerializer.Serialize(telemetryItems, true);
// Instantiate Transmission with the mock HttpClient
Transmission transmission = new Transmission(testUri, serializedData, fakeHttpClient, string.Empty, string.Empty);
// VALIDATE
transmission.TransmissionStatusEvent += delegate(object sender, TransmissionStatusEventArgs args)
{
var sendertransmission = sender as Transmission;
// convert raw JSON response to Backendresponse object
BackendResponse backendResponse = GetBackendResponse(args.Response.Content);
// Deserialize telemetry items to identify which items has failed
string[] items = JsonSerializer
.Deserialize(sendertransmission.Content)
.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
string[] failedItems = new string[3];
int i = 0;
// Create a list of failed items
foreach (var error in backendResponse.Errors)
{
failedItems[i++] = items[error.Index];
}
Assert.AreEqual((int)HttpStatusCode.PartialContent, args.Response.StatusCode);
Assert.AreEqual(5, backendResponse.ItemsReceived);
Assert.AreEqual(2, backendResponse.ItemsAccepted);
//IKEY_1
int totalItemsForIkey = items.Where(x => x.Contains("IKEY_1")).Count();
int failedItemsForIkey = failedItems.Where(x => x.Contains("IKEY_1")).Count();
Assert.AreEqual(2, totalItemsForIkey);
Assert.AreEqual(1, failedItemsForIkey);
//IKEY_2
totalItemsForIkey = items.Where(x => x.Contains("IKEY_2")).Count();
failedItemsForIkey = failedItems.Where(x => x.Contains("IKEY_2")).Count();
Assert.AreEqual(2, totalItemsForIkey);
Assert.AreEqual(1, failedItemsForIkey);
//IKEY_3
totalItemsForIkey = items.Where(x => x.Contains("IKEY_3")).Count();
failedItemsForIkey = failedItems.Where(x => x.Contains("IKEY_3")).Count();
Assert.AreEqual(1, totalItemsForIkey);
Assert.AreEqual(1, failedItemsForIkey);
};
// ACT
HttpWebResponseWrapper result = await transmission.SendAsync();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TransmissionStatusEventArgs"/> class.
/// </summary>
/// <param name="response">Response from ingestion endpoint.</param>
/// <param name="responseDurationInMs">Response duration in milliseconds.</param>
public TransmissionStatusEventArgs(HttpWebResponseWrapper response, long responseDurationInMs)
{
this.Response = response;
this.ResponseDurationInMs = responseDurationInMs;
}
public TransmissionStatusEventArgs(HttpWebResponseWrapper response) : this(response, default)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="TransmissionStatusEventArgs"/> class.
/// </summary>
/// <param name="response">Response from ingestion endpoint.</param>
public TransmissionStatusEventArgs(HttpWebResponseWrapper response)
{
this.Response = response;
}
