public async Task Invoke(IIncomingGrainCallContext context) { bool isOrleansGrain = context.InterfaceMethod == null || context.InterfaceMethod.DeclaringType == null || context.InterfaceMethod.Module.Assembly.FullName.StartsWith("Orleans"); //TODO add test that validate that we are not introducing new grain in micro dot bool isMicrodotGrain = isOrleansGrain == false && context.InterfaceMethod.DeclaringType.Name == nameof(IRequestProcessingGrain); bool isServiceGrain = isOrleansGrain == false && isMicrodotGrain == false; var grainTags = new Lazy <GrainTags>(() => new GrainTags(context)); // Drop the request if we're overloaded var loadSheddingConfig = _loadSheddingConfig(); if ( (loadSheddingConfig.ApplyToMicrodotGrains && isMicrodotGrain) || (loadSheddingConfig.ApplyToServiceGrains && isServiceGrain) ) { //Can brake the flow by throwing Overloaded RejectRequestIfLateOrOverloaded(grainTags); } var loggingConfig = _grainLoggingConfig(); bool shouldLog = (loggingConfig.LogOrleansGrains && isOrleansGrain) || (loggingConfig.LogMicrodotGrains && isMicrodotGrain) || (loggingConfig.LogServiceGrains && isServiceGrain); shouldLog = shouldLog && !ShouldSkipLoggingUnderRatio(loggingConfig, TracingContext.TryGetRequestID()); GrainCallEvent grainEvent = null; if (shouldLog) { RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call. RequestTimings.Current.Request.Start(); grainEvent = _eventPublisher.CreateEvent(); grainEvent.ParentSpanId = TracingContext.TryGetParentSpanID(); grainEvent.SpanId = Guid.NewGuid().ToString("N"); TracingContext.SetParentSpan(grainEvent.SpanId); } Exception ex = null; try { await context.Invoke(); } catch (Exception e) { ex = e; throw; } finally { if (shouldLog) { RequestTimings.Current.Request.Stop(); PublishEvent(ex, grainTags, grainEvent); } } }
private async Task <object> InvokeCore(HttpServiceRequest request, Type resultReturnType, JsonSerializerSettings jsonSettings) { if (request == null) { throw new ArgumentNullException(nameof(request)); } request.TracingData = new TracingData { HostName = CurrentApplicationInfo.HostName?.ToUpperInvariant(), ServiceName = AppInfo.Name, RequestID = TracingContext.TryGetRequestID(), SpanID = Guid.NewGuid().ToString("N"), //Each call is new span ParentSpanID = TracingContext.TryGetParentSpanID(), SpanStartTime = DateTimeOffset.UtcNow, AbandonRequestBy = TracingContext.AbandonRequestBy }; PrepareRequest?.Invoke(request); while (true) { var config = GetConfig(); var clientCallEvent = EventPublisher.CreateEvent(); clientCallEvent.TargetService = ServiceName; clientCallEvent.RequestId = request.TracingData?.RequestID; clientCallEvent.TargetMethod = request.Target.MethodName; clientCallEvent.SpanId = request.TracingData?.SpanID; clientCallEvent.ParentSpanId = request.TracingData?.ParentSpanID; string responseContent; HttpResponseMessage response; var nodeAndLoadBalancer = await ServiceDiscovery.GetNode().ConfigureAwait(false); // can throw int?effectivePort = GetEffectivePort(nodeAndLoadBalancer.Node, config); if (effectivePort == null) { throw new ConfigurationException("Cannot access service. Service Port not configured. See tags to find missing configuration", unencrypted: new Tags { { "ServiceName", ServiceName }, { "Required configuration key", $"Discovery.{ServiceName}.DefaultPort" } }); } // The URL is only for a nice experience in Fiddler, it's never parsed/used for anything. var uri = BuildUri(nodeAndLoadBalancer.Node.Hostname, effectivePort.Value, config) + ServiceName; if (request.Target.MethodName != null) { uri += $".{request.Target.MethodName}"; } if (request.Target.Endpoint != null) { uri += $"/{request.Target.Endpoint}"; } try { Log.Debug(_ => _("ServiceProxy: Calling remote service. See tags for details.", unencryptedTags: new { remoteEndpoint = nodeAndLoadBalancer.Node.Hostname, remotePort = effectivePort, remoteServiceName = ServiceName, remoteMethodName = request.Target.MethodName })); clientCallEvent.TargetHostName = nodeAndLoadBalancer.Node.Hostname; clientCallEvent.TargetPort = effectivePort.Value; clientCallEvent.TargetEnvironment = nodeAndLoadBalancer.TargetEnvironment; request.Overrides = TracingContext.TryGetOverrides()?.ShallowCloneWithDifferentPreferredEnvironment(nodeAndLoadBalancer.PreferredEnvironment) ?? new RequestOverrides { PreferredEnvironment = nodeAndLoadBalancer.PreferredEnvironment }; string requestContent = _serializationTime.Time(() => JsonConvert.SerializeObject(request, jsonSettings)); var httpContent = new StringContent(requestContent, Encoding.UTF8, "application/json"); httpContent.Headers.Add(GigyaHttpHeaders.ProtocolVersion, HttpServiceRequest.ProtocolVersion); clientCallEvent.RequestStartTimestamp = Stopwatch.GetTimestamp(); try { response = await GetHttpClient(config).PostAsync(uri, httpContent).ConfigureAwait(false); responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } finally { clientCallEvent.ResponseEndTimestamp = Stopwatch.GetTimestamp(); } if (response.Headers.TryGetValues(GigyaHttpHeaders.ExecutionTime, out IEnumerable <string> values)) { var time = values.FirstOrDefault(); if (TimeSpan.TryParse(time, out TimeSpan executionTime)) { clientCallEvent.ServerTimeMs = executionTime.TotalMilliseconds; } } } catch (HttpRequestException ex) { Log.Error("The remote service failed to return a valid HTTP response. Continuing to next " + "host. See tags for URL and exception for details.", exception: ex, unencryptedTags: new { uri }); _hostFailureCounter.Increment("RequestFailure"); clientCallEvent.Exception = ex; EventPublisher.TryPublish(clientCallEvent); // fire and forget! if (nodeAndLoadBalancer.LoadBalancer != null) { nodeAndLoadBalancer.LoadBalancer.ReportUnreachable(nodeAndLoadBalancer.Node, ex); continue; } throw; } catch (TaskCanceledException ex) { _failureCounter.Increment("RequestTimeout"); Exception rex = new RemoteServiceException("The request to the remote service exceeded the " + "allotted timeout. See the 'RequestUri' property on this exception for the URL that was " + "called and the tag 'requestTimeout' for the configured timeout.", uri, ex, unencrypted: new Tags { { "requestTimeout", LastHttpClient?.Timeout.ToString() }, { "requestUri", uri } }); clientCallEvent.Exception = rex; EventPublisher.TryPublish(clientCallEvent); // fire and forget! throw rex; } if (response.Headers.Contains(GigyaHttpHeaders.ServerHostname) || response.Headers.Contains(GigyaHttpHeaders.ProtocolVersion)) { try { if (response.IsSuccessStatusCode) { var returnObj = _deserializationTime.Time(() => JsonConvert.DeserializeObject(responseContent, resultReturnType, jsonSettings)); clientCallEvent.ErrCode = 0; EventPublisher.TryPublish(clientCallEvent); // fire and forget! _successCounter.Increment(); return(returnObj); } else { Exception remoteException; try { remoteException = _deserializationTime.Time(() => ExceptionSerializer.Deserialize(responseContent)); } catch (Exception ex) { _applicationExceptionCounter.Increment("ExceptionDeserializationFailure"); throw new RemoteServiceException("The remote service returned a failure response " + "that failed to deserialize. See the 'RequestUri' property on this exception " + "for the URL that was called, the inner exception for the exact error and the " + "'responseContent' encrypted tag for the original response content.", uri, ex, unencrypted: new Tags { { "requestUri", uri } }, encrypted: new Tags { { "responseContent", responseContent } }); } _applicationExceptionCounter.Increment(); clientCallEvent.Exception = remoteException; EventPublisher.TryPublish(clientCallEvent); // fire and forget! if (remoteException is RequestException || remoteException is EnvironmentException) { ExceptionDispatchInfo.Capture(remoteException).Throw(); } if (remoteException is UnhandledException) { remoteException = remoteException.InnerException; } throw new RemoteServiceException("The remote service returned a failure response. See " + "the 'RequestUri' property on this exception for the URL that was called, and the " + "inner exception for details.", uri, remoteException, unencrypted: new Tags { { "requestUri", uri } }); } } catch (JsonException ex) { _failureCounter.Increment("Serialization"); Log.Error("The remote service returned a response with JSON that failed " + "deserialization. See the 'uri' tag for the URL that was called, the exception for the " + "exact error and the 'responseContent' encrypted tag for the original response content.", exception: ex, unencryptedTags: new { uri }, encryptedTags: new { responseContent }); clientCallEvent.Exception = ex; EventPublisher.TryPublish(clientCallEvent); // fire and forget! throw new RemoteServiceException("The remote service returned a response with JSON that " + "failed deserialization. See the 'RequestUri' property on this exception for the URL " + "that was called, the inner exception for the exact error and the 'responseContent' " + "encrypted tag for the original response content.", uri, ex, new Tags { { "responseContent", responseContent } }, new Tags { { "requestUri", uri } }); } } else { var exception = response.StatusCode == HttpStatusCode.ServiceUnavailable ? new Exception($"The remote service is unavailable (503) and is not recognized as a Gigya host at uri: {uri}") : new Exception($"The remote service returned a response but is not recognized as a Gigya host at uri: {uri}"); if (nodeAndLoadBalancer.LoadBalancer == null) { throw exception; } nodeAndLoadBalancer.LoadBalancer.ReportUnreachable(nodeAndLoadBalancer.Node, exception); _hostFailureCounter.Increment("NotGigyaHost"); if (response.StatusCode == HttpStatusCode.ServiceUnavailable) { Log.Error("The remote service is unavailable (503) and is not recognized as a Gigya host. Continuing to next host.", unencryptedTags: new { uri }); } else { Log.Error("The remote service returned a response but is not recognized as a Gigya host. Continuing to next host.", unencryptedTags: new { uri, statusCode = response.StatusCode }, encryptedTags: new { responseContent }); } clientCallEvent.ErrCode = 500001; //(int)GSErrors.General_Server_Error; EventPublisher.TryPublish(clientCallEvent); // fire and forget! } } }