private async Task <object> Interceptor(MethodInfo targetMethod, InvokeMethodRequest request, IGrain target, IGrainMethodInvoker invoker)
{
if (targetMethod == null)
{
throw new ArgumentNullException(nameof(targetMethod));
}
var declaringNameSpace = targetMethod.DeclaringType?.Namespace;
// Do not intercept Orleans grains or other grains which should not be included in statistics.
if (targetMethod.DeclaringType.GetCustomAttribute <ExcludeGrainFromStatisticsAttribute>() != null ||
declaringNameSpace?.StartsWith("Orleans") == true)
{
return(await invoker.Invoke(target, request));
}
RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call.
RequestTimings.Current.Request.Start();
Exception ex = null;
try
{
return(await invoker.Invoke(target, request));
}
catch (Exception e)
{
ex = e;
throw;
}
finally
{
RequestTimings.Current.Request.Stop();
var grainEvent = EventPublisher.CreateEvent();
grainEvent.TargetType = targetMethod.DeclaringType?.FullName;
grainEvent.TargetMethod = targetMethod.Name;
grainEvent.Exception = ex;
grainEvent.ErrCode = ex != null ? null : (int?)0;
try
{
EventPublisher.TryPublish(grainEvent);
}
catch (Exception)
{
EventsDiscarded.Increment();
}
}
}
private void PublishEvent(HttpServiceRequest requestData, Exception ex, ServiceMethod serviceMethod, double requestTime)
{
var callEvent = EventPublisher.CreateEvent();
callEvent.CalledServiceName = serviceMethod?.GrainInterfaceType.Name;
callEvent.ClientMetadata = requestData.TracingData;
callEvent.ServiceMethod = requestData.Target?.MethodName;
callEvent.Params = (requestData.Arguments ?? new OrderedDictionary()).Cast <DictionaryEntry>().Select(arg => new Param
{
Name = arg.Key.ToString(),
Value = arg.Value is string?arg.Value.ToString() : JsonConvert.SerializeObject(arg.Value)
});
callEvent.Exception = ex;
callEvent.ActualTotalTime = requestTime;
callEvent.ErrCode = ex != null ? null : (int?)0;
EventPublisher.TryPublish(callEvent); // fire and forget!
}
private void PublishEvent(MethodInfo targetMethod, IGrain target, Exception ex)
{
var grainEvent = EventPublisher.CreateEvent();
if (target.GetPrimaryKeyString() != null)
{
grainEvent.GrainKeyString = target.GetPrimaryKeyString();
}
else if (target.IsPrimaryKeyBasedOnLong())
{
grainEvent.GrainKeyLong = target.GetPrimaryKeyLong(out var keyExt);
grainEvent.GrainKeyExtention = keyExt;
}
else
{
grainEvent.GrainKeyGuid = target.GetPrimaryKey(out var keyExt);
grainEvent.GrainKeyExtention = keyExt;
}
if (target is Grain grainTarget)
{
grainEvent.SiloAddress = grainTarget.RuntimeIdentity;
}
grainEvent.SiloDeploymentId = ConfigBuilder.ClusterConfiguration.Globals.DeploymentId;
grainEvent.TargetType = targetMethod.DeclaringType?.FullName;
grainEvent.TargetMethod = targetMethod.Name;
grainEvent.Exception = ex;
grainEvent.ErrCode = ex != null ? null : (int?)0;
try
{
EventPublisher.TryPublish(grainEvent);
}
catch (Exception)
{
EventsDiscarded.Increment();
}
}
private async Task <object> Interceptor(MethodInfo targetMethod, InvokeMethodRequest request, IGrain target, IGrainMethodInvoker invoker)
{
if (targetMethod == null)
{
throw new ArgumentNullException(nameof(targetMethod));
}
var declaringNameSpace = targetMethod.DeclaringType?.Namespace;
// Do not intercept Orleans grains or other grains which should not be included in statistics.
if (targetMethod.DeclaringType.GetCustomAttribute <ExcludeGrainFromStatisticsAttribute>() != null ||
declaringNameSpace?.StartsWith("Orleans") == true)
{
return(await invoker.Invoke(target, request));
}
RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call.
RequestTimings.Current.Request.Start();
Exception ex = null;
try
{
return(await invoker.Invoke(target, request));
}
catch (Exception e)
{
ex = e;
throw;
}
finally
{
RequestTimings.Current.Request.Stop();
var grainEvent = EventPublisher.CreateEvent();
if (target.GetPrimaryKeyString() != null)
{
grainEvent.GrainKeyString = target.GetPrimaryKeyString();
}
else if (target.IsPrimaryKeyBasedOnLong())
{
grainEvent.GrainKeyLong = target.GetPrimaryKeyLong(out var keyExt);
grainEvent.GrainKeyExtention = keyExt;
}
else
{
grainEvent.GrainKeyGuid = target.GetPrimaryKey(out var keyExt);
grainEvent.GrainKeyExtention = keyExt;
}
if (target is Grain grainTarget)
{
grainEvent.SiloAddress = grainTarget.RuntimeIdentity;
}
grainEvent.SiloDeploymentId = ConfigBuilder.ClusterConfiguration.Globals.DeploymentId;
grainEvent.TargetType = targetMethod.DeclaringType?.FullName;
grainEvent.TargetMethod = targetMethod.Name;
grainEvent.Exception = ex;
grainEvent.ErrCode = ex != null ? null : (int?)0;
try
{
EventPublisher.TryPublish(grainEvent);
}
catch (Exception)
{
EventsDiscarded.Increment();
}
}
}
private async Task <object> InvokeCore(HttpServiceRequest request, Type resultReturnType, JsonSerializerSettings jsonSettings)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
request.TracingData = new TracingData
{
HostName = CurrentApplicationInfo.HostName?.ToUpperInvariant(),
ServiceName = CurrentApplicationInfo.Name,
RequestID = TracingContext.TryGetRequestID(),
SpanID = Guid.NewGuid().ToString("N"), //Each call is new span
ParentSpanID = TracingContext.TryGetSpanID(),
SpanStartTime = DateTimeOffset.UtcNow,
AbandonRequestBy = TracingContext.AbandonRequestBy
};
PrepareRequest?.Invoke(request);
while (true)
{
var config = GetConfig();
var clientCallEvent = EventPublisher.CreateEvent();
clientCallEvent.TargetService = ServiceName;
clientCallEvent.RequestId = request.TracingData?.RequestID;
clientCallEvent.TargetMethod = request.Target.MethodName;
clientCallEvent.SpanId = request.TracingData?.SpanID;
clientCallEvent.ParentSpanId = request.TracingData?.ParentSpanID;
string responseContent;
HttpResponseMessage response;
var nodeAndLoadBalancer = await ServiceDiscovery.GetNode().ConfigureAwait(false); // can throw
int?effectivePort = GetEffectivePort(nodeAndLoadBalancer.Node, config);
if (effectivePort == null)
{
throw new ConfigurationException("Cannot access service. Service Port not configured. See tags to find missing configuration", unencrypted: new Tags {
{ "ServiceName", ServiceName },
{ "Required configuration key", $"Discovery.{ServiceName}.DefaultPort" }
});
}
// The URL is only for a nice experience in Fiddler, it's never parsed/used for anything.
var uri = BuildUri(nodeAndLoadBalancer.Node.Hostname, effectivePort.Value, config) + ServiceName;
if (request.Target.MethodName != null)
{
uri += $".{request.Target.MethodName}";
}
if (request.Target.Endpoint != null)
{
uri += $"/{request.Target.Endpoint}";
}
try
{
Log.Debug(_ => _("ServiceProxy: Calling remote service. See tags for details.",
unencryptedTags: new
{
remoteEndpoint = nodeAndLoadBalancer.Node.Hostname,
remotePort = effectivePort,
remoteServiceName = ServiceName,
remoteMethodName = request.Target.MethodName
}));
clientCallEvent.TargetHostName = nodeAndLoadBalancer.Node.Hostname;
clientCallEvent.TargetPort = effectivePort.Value;
request.Overrides = TracingContext.TryGetOverrides()?.ShallowCloneWithDifferentPreferredEnvironment(nodeAndLoadBalancer.PreferredEnvironment)
?? new RequestOverrides {
PreferredEnvironment = nodeAndLoadBalancer.PreferredEnvironment
};
string requestContent = _serializationTime.Time(() => JsonConvert.SerializeObject(request, jsonSettings));
var httpContent = new StringContent(requestContent, Encoding.UTF8, "application/json");
httpContent.Headers.Add(GigyaHttpHeaders.ProtocolVersion, HttpServiceRequest.ProtocolVersion);
clientCallEvent.RequestStartTimestamp = Stopwatch.GetTimestamp();
try
{
response = await GetHttpClient(config).PostAsync(uri, httpContent).ConfigureAwait(false);
responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
finally
{
clientCallEvent.ResponseEndTimestamp = Stopwatch.GetTimestamp();
}
if (response.Headers.TryGetValues(GigyaHttpHeaders.ExecutionTime, out IEnumerable <string> values))
{
var time = values.FirstOrDefault();
if (TimeSpan.TryParse(time, out TimeSpan executionTime))
{
clientCallEvent.ServerTimeMs = executionTime.TotalMilliseconds;
}
}
}
catch (HttpRequestException ex)
{
Log.Error("The remote service failed to return a valid HTTP response. Continuing to next " +
"host. See tags for URL and exception for details.",
exception: ex,
unencryptedTags: new { uri });
_hostFailureCounter.Increment("RequestFailure");
clientCallEvent.Exception = ex;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
if (nodeAndLoadBalancer.LoadBalancer != null)
{
nodeAndLoadBalancer.LoadBalancer.ReportUnreachable(nodeAndLoadBalancer.Node, ex);
continue;
}
throw;
}
catch (TaskCanceledException ex)
{
_failureCounter.Increment("RequestTimeout");
Exception rex = new RemoteServiceException("The request to the remote service exceeded the " +
"allotted timeout. See the 'RequestUri' property on this exception for the URL that was " +
"called and the tag 'requestTimeout' for the configured timeout.",
uri,
ex,
unencrypted: new Tags
{
{ "requestTimeout", LastHttpClient?.Timeout.ToString() },
{ "requestUri", uri }
});
clientCallEvent.Exception = rex;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
throw rex;
}
if (response.Headers.Contains(GigyaHttpHeaders.ServerHostname) || response.Headers.Contains(GigyaHttpHeaders.ProtocolVersion))
{
try
{
if (response.IsSuccessStatusCode)
{
var returnObj = _deserializationTime.Time(() => JsonConvert.DeserializeObject(responseContent, resultReturnType, jsonSettings));
clientCallEvent.ErrCode = 0;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
_successCounter.Increment();
return(returnObj);
}
else
{
Exception remoteException;
try
{
remoteException = _deserializationTime.Time(() => ExceptionSerializer.Deserialize(responseContent));
}
catch (Exception ex)
{
_applicationExceptionCounter.Increment("ExceptionDeserializationFailure");
throw new RemoteServiceException("The remote service returned a failure response " +
"that failed to deserialize. See the 'RequestUri' property on this exception " +
"for the URL that was called, the inner exception for the exact error and the " +
"'responseContent' encrypted tag for the original response content.",
uri,
ex,
unencrypted: new Tags {
{ "requestUri", uri }
},
encrypted: new Tags {
{ "responseContent", responseContent }
});
}
_applicationExceptionCounter.Increment();
clientCallEvent.Exception = remoteException;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
if (remoteException is RequestException || remoteException is EnvironmentException)
{
ExceptionDispatchInfo.Capture(remoteException).Throw();
}
if (remoteException is UnhandledException)
{
remoteException = remoteException.InnerException;
}
throw new RemoteServiceException("The remote service returned a failure response. See " +
"the 'RequestUri' property on this exception for the URL that was called, and the " +
"inner exception for details.",
uri,
remoteException,
unencrypted: new Tags {
{ "requestUri", uri }
});
}
}
catch (JsonException ex)
{
_failureCounter.Increment("Serialization");
Log.Error("The remote service returned a response with JSON that failed " +
"deserialization. See the 'uri' tag for the URL that was called, the exception for the " +
"exact error and the 'responseContent' encrypted tag for the original response content.",
exception: ex,
unencryptedTags: new { uri },
encryptedTags: new { responseContent });
clientCallEvent.Exception = ex;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
throw new RemoteServiceException("The remote service returned a response with JSON that " +
"failed deserialization. See the 'RequestUri' property on this exception for the URL " +
"that was called, the inner exception for the exact error and the 'responseContent' " +
"encrypted tag for the original response content.",
uri,
ex,
new Tags {
{ "responseContent", responseContent }
},
new Tags {
{ "requestUri", uri }
});
}
}
else
{
var exception = response.StatusCode == HttpStatusCode.ServiceUnavailable ?
new Exception($"The remote service is unavailable (503) and is not recognized as a Gigya host at uri: {uri}") :
new Exception($"The remote service returned a response but is not recognized as a Gigya host at uri: {uri}");
nodeAndLoadBalancer.LoadBalancer.ReportUnreachable(nodeAndLoadBalancer.Node, exception);
_hostFailureCounter.Increment("NotGigyaHost");
if (response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
Log.Error("The remote service is unavailable (503) and is not recognized as a Gigya host. Continuing to next host.", unencryptedTags: new { uri });
}
else
{
Log.Error("The remote service returned a response but is not recognized as a Gigya host. Continuing to next host.", unencryptedTags: new { uri, statusCode = response.StatusCode }, encryptedTags: new { responseContent });
}
clientCallEvent.ErrCode = 500001; //(int)GSErrors.General_Server_Error;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
}
}
}
