public async Task Invoke(IIncomingGrainCallContext context)
{
bool isOrleansGrain = context.InterfaceMethod == null || context.InterfaceMethod.DeclaringType == null || context.InterfaceMethod.Module.Assembly.FullName.StartsWith("Orleans");
//TODO add test that validate that we are not introducing new grain in micro dot
bool isMicrodotGrain = isOrleansGrain == false && context.InterfaceMethod.DeclaringType.Name == nameof(IRequestProcessingGrain);
bool isServiceGrain = isOrleansGrain == false && isMicrodotGrain == false;
var grainTags = new Lazy <GrainTags>(() => new GrainTags(context));
// Drop the request if we're overloaded
var loadSheddingConfig = _loadSheddingConfig();
if (
(loadSheddingConfig.ApplyToMicrodotGrains && isMicrodotGrain) ||
(loadSheddingConfig.ApplyToServiceGrains && isServiceGrain)
)
{
//Can brake the flow by throwing Overloaded
RejectRequestIfLateOrOverloaded(grainTags);
}
var loggingConfig = _grainLoggingConfig();
bool shouldLog = (loggingConfig.LogOrleansGrains && isOrleansGrain) ||
(loggingConfig.LogMicrodotGrains && isMicrodotGrain) ||
(loggingConfig.LogServiceGrains && isServiceGrain);
shouldLog = shouldLog && !ShouldSkipLoggingUnderRatio(loggingConfig, TracingContext.TryGetRequestID());
GrainCallEvent grainEvent = null;
if (shouldLog)
{
RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call.
RequestTimings.Current.Request.Start();
grainEvent = _eventPublisher.CreateEvent();
grainEvent.ParentSpanId = TracingContext.TryGetParentSpanID();
grainEvent.SpanId = Guid.NewGuid().ToString("N");
TracingContext.SetParentSpan(grainEvent.SpanId);
}
Exception ex = null;
try
{
await context.Invoke();
}
catch (Exception e)
{
ex = e;
throw;
}
finally
{
if (shouldLog)
{
RequestTimings.Current.Request.Stop();
PublishEvent(ex, grainTags, grainEvent);
}
}
}
private uint GetIndexByTrafficRoutingStrategy()
{
switch (TrafficRoutingStrategy)
{
case TrafficRoutingStrategy.RoundRobin:
return((uint)Interlocked.Increment(ref _roundRobinIndex));
case TrafficRoutingStrategy.RandomByRequestID:
return((uint?)TracingContext.TryGetRequestID()?.GetHashCode() ?? (uint)Interlocked.Increment(ref _roundRobinIndex));
default:
throw new ProgrammaticException($"The {nameof(TrafficRoutingStrategy)} '{TrafficRoutingStrategy}' is not supported by LoadBalancer.");
}
}
public async Task SingleGrainCall_CallSucceeds_PublishesEvent()
{
_flumeQueue.Clear();
var requestId = nameof(SingleGrainCall_CallSucceeds_PublishesEvent) + Guid.NewGuid();
TracingContext.SetRequestID(requestId);
TracingContext.TryGetRequestID();
await _serviceProxy.Add(5, 3);
var events = _flumeQueue.Events;
var grainReq = events.Where(r => r.EventType == "grainReq")
.Select(r => (GrainCallEvent)r)
.Single(r => r.TargetType == typeof(CalculatorServiceGrain).FullName);
Assert.AreEqual("Add", grainReq.TargetMethod);
Assert.AreEqual(requestId, grainReq.RequestId);
}
public async Task SingleServerCall_CallSucceeds_PublishesEvent()
{
_flumeQueue.Clear();
var requestId = nameof(SingleServerCall_CallSucceeds_PublishesEvent) + Guid.NewGuid();
TracingContext.SetRequestID(requestId);
TracingContext.TryGetRequestID();
await _serviceProxy.Add(5, 3);
await Task.Delay(100);
var events = _flumeQueue.Events;
var serverReq = (ServiceCallEvent)events.Single();
Assert.AreEqual("serverReq", serverReq.EventType);
Assert.AreEqual(nameof(ICalculatorService), serverReq.ServiceName);
Assert.AreEqual("Add", serverReq.ServiceMethod);
Assert.AreEqual(requestId, serverReq.RequestId);
}
private async Task <object> InvokeCore(HttpServiceRequest request, Type resultReturnType, JsonSerializerSettings jsonSettings)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
request.TracingData = new TracingData
{
HostName = CurrentApplicationInfo.HostName?.ToUpperInvariant(),
ServiceName = CurrentApplicationInfo.Name,
RequestID = TracingContext.TryGetRequestID(),
SpanID = Guid.NewGuid().ToString("N"), //Each call is new span
ParentSpanID = TracingContext.TryGetSpanID(),
SpanStartTime = DateTimeOffset.UtcNow,
AbandonRequestBy = TracingContext.AbandonRequestBy
};
PrepareRequest?.Invoke(request);
while (true)
{
var config = GetConfig();
var clientCallEvent = EventPublisher.CreateEvent();
clientCallEvent.TargetService = ServiceName;
clientCallEvent.RequestId = request.TracingData?.RequestID;
clientCallEvent.TargetMethod = request.Target.MethodName;
clientCallEvent.SpanId = request.TracingData?.SpanID;
clientCallEvent.ParentSpanId = request.TracingData?.ParentSpanID;
string responseContent;
HttpResponseMessage response;
var nodeAndLoadBalancer = await ServiceDiscovery.GetNode().ConfigureAwait(false); // can throw
int?effectivePort = GetEffectivePort(nodeAndLoadBalancer.Node, config);
if (effectivePort == null)
{
throw new ConfigurationException("Cannot access service. Service Port not configured. See tags to find missing configuration", unencrypted: new Tags {
{ "ServiceName", ServiceName },
{ "Required configuration key", $"Discovery.{ServiceName}.DefaultPort" }
});
}
// The URL is only for a nice experience in Fiddler, it's never parsed/used for anything.
var uri = BuildUri(nodeAndLoadBalancer.Node.Hostname, effectivePort.Value, config) + ServiceName;
if (request.Target.MethodName != null)
{
uri += $".{request.Target.MethodName}";
}
if (request.Target.Endpoint != null)
{
uri += $"/{request.Target.Endpoint}";
}
try
{
Log.Debug(_ => _("ServiceProxy: Calling remote service. See tags for details.",
unencryptedTags: new
{
remoteEndpoint = nodeAndLoadBalancer.Node.Hostname,
remotePort = effectivePort,
remoteServiceName = ServiceName,
remoteMethodName = request.Target.MethodName
}));
clientCallEvent.TargetHostName = nodeAndLoadBalancer.Node.Hostname;
clientCallEvent.TargetPort = effectivePort.Value;
request.Overrides = TracingContext.TryGetOverrides()?.ShallowCloneWithDifferentPreferredEnvironment(nodeAndLoadBalancer.PreferredEnvironment)
?? new RequestOverrides {
PreferredEnvironment = nodeAndLoadBalancer.PreferredEnvironment
};
string requestContent = _serializationTime.Time(() => JsonConvert.SerializeObject(request, jsonSettings));
var httpContent = new StringContent(requestContent, Encoding.UTF8, "application/json");
httpContent.Headers.Add(GigyaHttpHeaders.ProtocolVersion, HttpServiceRequest.ProtocolVersion);
clientCallEvent.RequestStartTimestamp = Stopwatch.GetTimestamp();
try
{
response = await GetHttpClient(config).PostAsync(uri, httpContent).ConfigureAwait(false);
responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
finally
{
clientCallEvent.ResponseEndTimestamp = Stopwatch.GetTimestamp();
}
if (response.Headers.TryGetValues(GigyaHttpHeaders.ExecutionTime, out IEnumerable <string> values))
{
var time = values.FirstOrDefault();
if (TimeSpan.TryParse(time, out TimeSpan executionTime))
{
clientCallEvent.ServerTimeMs = executionTime.TotalMilliseconds;
}
}
}
catch (HttpRequestException ex)
{
Log.Error("The remote service failed to return a valid HTTP response. Continuing to next " +
"host. See tags for URL and exception for details.",
exception: ex,
unencryptedTags: new { uri });
_hostFailureCounter.Increment("RequestFailure");
clientCallEvent.Exception = ex;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
if (nodeAndLoadBalancer.LoadBalancer != null)
{
nodeAndLoadBalancer.LoadBalancer.ReportUnreachable(nodeAndLoadBalancer.Node, ex);
continue;
}
throw;
}
catch (TaskCanceledException ex)
{
_failureCounter.Increment("RequestTimeout");
Exception rex = new RemoteServiceException("The request to the remote service exceeded the " +
"allotted timeout. See the 'RequestUri' property on this exception for the URL that was " +
"called and the tag 'requestTimeout' for the configured timeout.",
uri,
ex,
unencrypted: new Tags
{
{ "requestTimeout", LastHttpClient?.Timeout.ToString() },
{ "requestUri", uri }
});
clientCallEvent.Exception = rex;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
throw rex;
}
if (response.Headers.Contains(GigyaHttpHeaders.ServerHostname) || response.Headers.Contains(GigyaHttpHeaders.ProtocolVersion))
{
try
{
if (response.IsSuccessStatusCode)
{
var returnObj = _deserializationTime.Time(() => JsonConvert.DeserializeObject(responseContent, resultReturnType, jsonSettings));
clientCallEvent.ErrCode = 0;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
_successCounter.Increment();
return(returnObj);
}
else
{
Exception remoteException;
try
{
remoteException = _deserializationTime.Time(() => ExceptionSerializer.Deserialize(responseContent));
}
catch (Exception ex)
{
_applicationExceptionCounter.Increment("ExceptionDeserializationFailure");
throw new RemoteServiceException("The remote service returned a failure response " +
"that failed to deserialize. See the 'RequestUri' property on this exception " +
"for the URL that was called, the inner exception for the exact error and the " +
"'responseContent' encrypted tag for the original response content.",
uri,
ex,
unencrypted: new Tags {
{ "requestUri", uri }
},
encrypted: new Tags {
{ "responseContent", responseContent }
});
}
_applicationExceptionCounter.Increment();
clientCallEvent.Exception = remoteException;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
if (remoteException is RequestException || remoteException is EnvironmentException)
{
ExceptionDispatchInfo.Capture(remoteException).Throw();
}
if (remoteException is UnhandledException)
{
remoteException = remoteException.InnerException;
}
throw new RemoteServiceException("The remote service returned a failure response. See " +
"the 'RequestUri' property on this exception for the URL that was called, and the " +
"inner exception for details.",
uri,
remoteException,
unencrypted: new Tags {
{ "requestUri", uri }
});
}
}
catch (JsonException ex)
{
_failureCounter.Increment("Serialization");
Log.Error("The remote service returned a response with JSON that failed " +
"deserialization. See the 'uri' tag for the URL that was called, the exception for the " +
"exact error and the 'responseContent' encrypted tag for the original response content.",
exception: ex,
unencryptedTags: new { uri },
encryptedTags: new { responseContent });
clientCallEvent.Exception = ex;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
throw new RemoteServiceException("The remote service returned a response with JSON that " +
"failed deserialization. See the 'RequestUri' property on this exception for the URL " +
"that was called, the inner exception for the exact error and the 'responseContent' " +
"encrypted tag for the original response content.",
uri,
ex,
new Tags {
{ "responseContent", responseContent }
},
new Tags {
{ "requestUri", uri }
});
}
}
else
{
var exception = response.StatusCode == HttpStatusCode.ServiceUnavailable ?
new Exception($"The remote service is unavailable (503) and is not recognized as a Gigya host at uri: {uri}") :
new Exception($"The remote service returned a response but is not recognized as a Gigya host at uri: {uri}");
nodeAndLoadBalancer.LoadBalancer.ReportUnreachable(nodeAndLoadBalancer.Node, exception);
_hostFailureCounter.Increment("NotGigyaHost");
if (response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
Log.Error("The remote service is unavailable (503) and is not recognized as a Gigya host. Continuing to next host.", unencryptedTags: new { uri });
}
else
{
Log.Error("The remote service returned a response but is not recognized as a Gigya host. Continuing to next host.", unencryptedTags: new { uri, statusCode = response.StatusCode }, encryptedTags: new { responseContent });
}
clientCallEvent.ErrCode = 500001; //(int)GSErrors.General_Server_Error;
EventPublisher.TryPublish(clientCallEvent); // fire and forget!
}
}
}
public async Task Invoke(IIncomingGrainCallContext context)
{
if (context.InterfaceMethod == null || context.InterfaceMethod.DeclaringType == null)
{
// We observing behavior when no interface method
// We still don't want to prevent it happen.
await context.Invoke();
return;
}
// Identify the grain type
bool isOrleansGrain = context.InterfaceMethod.Module.Assembly.FullName?.StartsWith("Orleans") == true;
//TODO add test that validate that we are not introducing new grain in micro dot
bool isMicrodotGrain = context.InterfaceMethod.DeclaringType.Name == nameof(IRequestProcessingGrain);
bool isServiceGrain = isOrleansGrain == false && isMicrodotGrain == false;
// Drop the request if we're overloaded
var loadSheddingConfig = _loadSheddingConfig();
if (
(loadSheddingConfig.ApplyToMicrodotGrains && isMicrodotGrain) ||
(loadSheddingConfig.ApplyToServiceGrains && isServiceGrain)
)
{
RejectRequestIfLateOrOverloaded(context);
}
var loggingConfig = _grainLoggingConfig();
var shouldLoad = ((loggingConfig.LogOrleansGrains && isOrleansGrain) ||
(loggingConfig.LogMicrodotGrains && isMicrodotGrain) ||
(loggingConfig.LogServiceGrains && isServiceGrain));
if (shouldLoad == false)
{
await context.Invoke();
return;
}
string callId = TracingContext.TryGetRequestID();
uint max = (uint)Math.Round(loggingConfig.LogRatio * uint.MaxValue);
if (loggingConfig.LogRatio == 0 || callId == null || (uint)callId.GetHashCode() % uint.MaxValue > max)
{
await context.Invoke();
return;
}
RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call.
RequestTimings.Current.Request.Start();
Exception ex = null;
try
{
await context.Invoke();
}
catch (Exception e)
{
ex = e;
throw;
}
finally
{
RequestTimings.Current.Request.Stop();
PublishEvent(context, ex);
}
}