private async Task <object> InvokeCore(HttpServiceRequest request, Type resultReturnType, JsonSerializerSettings jsonSettings) { if (request == null) { throw new ArgumentNullException(nameof(request)); } request.Overrides = TracingContext.TryGetOverrides(); request.TracingData = new TracingData { HostName = CurrentApplicationInfo.HostName?.ToUpperInvariant(), ServiceName = CurrentApplicationInfo.Name, RequestID = TracingContext.TryGetRequestID(), SpanID = Guid.NewGuid().ToString("N"), //Each call is new span ParentSpanID = TracingContext.TryGetSpanID(), SpanStartTime = DateTimeOffset.UtcNow, AbandonRequestBy = TracingContext.AbandonRequestBy }; PrepareRequest?.Invoke(request); var requestContent = JsonConvert.SerializeObject(request, jsonSettings); while (true) { var config = GetConfig(); var clientCallEvent = EventPublisher.CreateEvent(); clientCallEvent.TargetService = ServiceName; clientCallEvent.RequestId = request.TracingData?.RequestID; clientCallEvent.TargetMethod = request.Target.MethodName; clientCallEvent.SpanId = request.TracingData?.SpanID; clientCallEvent.ParentSpanId = request.TracingData?.ParentSpanID; string responseContent; HttpResponseMessage response; IEndPointHandle endPoint = await ServiceDiscovery.GetNextHost(clientCallEvent.RequestId).ConfigureAwait(false); int?effectivePort = GetEffectivePort(endPoint, config); if (effectivePort == null) { throw new ConfigurationException("Cannot access service. Service Port not configured. See tags to find missing configuration", unencrypted: new Tags { { "ServiceName", ServiceName }, { "Required configuration key", $"Discovery.{ServiceName}.DefaultPort" } }); } // The URL is only for a nice experience in Fiddler, it's never parsed/used for anything. var uri = BuildUri(endPoint.HostName, effectivePort.Value, config) + ServiceName; if (request.Target.MethodName != null) { uri += $".{request.Target.MethodName}"; } if (request.Target.Endpoint != null) { uri += $"/{request.Target.Endpoint}"; } try { Log.Debug(_ => _("ServiceProxy: Calling remote service. See tags for details.", unencryptedTags: new { remoteEndpoint = endPoint.HostName, remotePort = effectivePort, remoteServiceName = ServiceName, remoteMethodName = request.Target.MethodName })); clientCallEvent.TargetHostName = endPoint.HostName; clientCallEvent.TargetPort = effectivePort.Value; var httpContent = new StringContent(requestContent, Encoding.UTF8, "application/json"); httpContent.Headers.Add(HSHttpHeaders.ProtocolVersion, HttpServiceRequest.ProtocolVersion); clientCallEvent.RequestStartTimestamp = Stopwatch.GetTimestamp(); try { response = await GetHttpClient(config).PostAsync(uri, httpContent).ConfigureAwait(false); responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } finally { clientCallEvent.ResponseEndTimestamp = Stopwatch.GetTimestamp(); } if (response.Headers.TryGetValues(HSHttpHeaders.ExecutionTime, out IEnumerable <string> values)) { var time = values.FirstOrDefault(); if (TimeSpan.TryParse(time, out TimeSpan executionTime)) { clientCallEvent.ServerTimeMs = executionTime.TotalMilliseconds; } } } catch (HttpRequestException ex) { Log.Error("The remote service failed to return a valid HTTP response. Continuing to next " + "host. See tags for URL and exception for details.", exception: ex, unencryptedTags: new { uri }); endPoint.ReportFailure(ex); clientCallEvent.Exception = ex; EventPublisher.TryPublish(clientCallEvent); // fire and forget! continue; } catch (TaskCanceledException ex) { Exception rex = new RemoteServiceException("The request to the remote service exceeded the " + "allotted timeout. See the 'RequestUri' property on this exception for the URL that was " + "called and the tag 'requestTimeout' for the configured timeout.", uri, ex, unencrypted: new Tags { { "requestTimeout", LastHttpClient?.Timeout.ToString() }, { "requestUri", uri } }); clientCallEvent.Exception = rex; EventPublisher.TryPublish(clientCallEvent); // fire and forget! throw rex; } if (response.Headers.Contains(HSHttpHeaders.ServerHostname) || response.Headers.Contains(HSHttpHeaders.ProtocolVersion)) { try { endPoint.ReportSuccess(); if (response.IsSuccessStatusCode) { var returnObj = JsonConvert.DeserializeObject(responseContent, resultReturnType, jsonSettings); clientCallEvent.ErrCode = 0; EventPublisher.TryPublish(clientCallEvent); // fire and forget! return(returnObj); } else { Exception remoteException; try { remoteException = ExceptionSerializer.Deserialize(responseContent); } catch (Exception ex) { throw new RemoteServiceException("The remote service returned a failure response " + "that failed to deserialize. See the 'RequestUri' property on this exception " + "for the URL that was called, the inner exception for the exact error and the " + "'responseContent' encrypted tag for the original response content.", uri, ex, unencrypted: new Tags { { "requestUri", uri } }, encrypted: new Tags { { "responseContent", responseContent } }); } clientCallEvent.Exception = remoteException; EventPublisher.TryPublish(clientCallEvent); // fire and forget! if (remoteException is RequestException || remoteException is EnvironmentException) { ExceptionDispatchInfo.Capture(remoteException).Throw(); } if (remoteException is UnhandledException) { remoteException = remoteException.InnerException; } throw new RemoteServiceException("The remote service returned a failure response. See " + "the 'RequestUri' property on this exception for the URL that was called, and the " + "inner exception for details.", uri, remoteException, unencrypted: new Tags { { "requestUri", uri } }); } } catch (JsonException ex) { Log.Error("The remote service returned a response with JSON that failed " + "deserialization. See the 'uri' tag for the URL that was called, the exception for the " + "exact error and the 'responseContent' encrypted tag for the original response content.", exception: ex, unencryptedTags: new { uri }, encryptedTags: new { responseContent }); clientCallEvent.Exception = ex; EventPublisher.TryPublish(clientCallEvent); // fire and forget! throw new RemoteServiceException("The remote service returned a response with JSON that " + "failed deserialization. See the 'RequestUri' property on this exception for the URL " + "that was called, the inner exception for the exact error and the 'responseContent' " + "encrypted tag for the original response content.", uri, ex, new Tags { { "responseContent", responseContent } }, new Tags { { "requestUri", uri } }); } } else { var exception = response.StatusCode == HttpStatusCode.ServiceUnavailable ? new Exception($"The remote service is unavailable (503) and is not recognized as a HS host at uri: {uri}") : new Exception($"The remote service returned a response but is not recognized as a HS host at uri: {uri}"); endPoint.ReportFailure(exception); if (response.StatusCode == HttpStatusCode.ServiceUnavailable) { Log.Error("The remote service is unavailable (503) and is not recognized as a HS host. Continuing to next host.", unencryptedTags: new { uri }); } else { Log.Error("The remote service returned a response but is not recognized as a HS host. Continuing to next host.", unencryptedTags: new { uri, statusCode = response.StatusCode }, encryptedTags: new { responseContent }); } clientCallEvent.ErrCode = 500001; //(int)GSErrors.General_Server_Error; EventPublisher.TryPublish(clientCallEvent); // fire and forget! } } }