public static void Add(
this IServiceCollection @this,
IServicePartition servicePartition)
{
if (@this == null)
{
throw new ArgumentNullException(nameof(@this));
}
if (servicePartition == null)
{
throw new ArgumentNullException(nameof(servicePartition));
}
@this.Add(new ServiceDescriptor(typeof(IServicePartition), servicePartition));
switch (servicePartition)
{
case IStatefulServicePartition _:
@this.Add(new ServiceDescriptor(typeof(IStatefulServicePartition), servicePartition));
break;
case IStatelessServicePartition _:
@this.Add(new ServiceDescriptor(typeof(IStatelessServicePartition), servicePartition));
break;
}
}
internal async Task AwaitRunAsyncWithHealthReporting(IServicePartition partition, Task runAsyncTask)
{
while (true)
{
var delayTaskCts = new CancellationTokenSource();
var delayTask = Task.Delay(RunAsyncExpectedCancellationTimeSpan, delayTaskCts.Token);
var finishedTask = await Task.WhenAny(runAsyncTask, delayTask);
if (finishedTask == runAsyncTask)
{
delayTaskCts.Cancel();
#pragma warning disable 4014
// Observe OperationCancelledException if any asynchronously.
delayTask.ContinueWith(t => t.Exception, TaskContinuationOptions.OnlyOnFaulted);
#pragma warning restore 4014
await runAsyncTask;
break;
}
// ReSharper disable once UseStringInterpolation
var msg = string.Format(
"RunAsync is taking longer then expected time ({0}s) to cancel.",
RunAsyncExpectedCancellationTimeSpan.TotalSeconds);
ServiceTrace.Source.WriteWarningWithId(this.traceType + ApiSlowTraceTypeSuffix, this.traceId, msg);
this.ReportRunAsyncSlowCancellationHealth(partition, msg);
}
}
public OrleansCommunicationListener(
StatelessServiceContext parameters,
ClusterConfiguration configuration,
IServicePartition servicePartition)
{
this.parameters = parameters;
this.configuration = configuration;
this.partition = servicePartition;
}
public OrleansCommunicationListener(
StatelessServiceContext parameters,
ClusterConfiguration configuration,
IServicePartition servicePartition)
{
this.parameters = parameters;
this.configuration = configuration;
this.partition = servicePartition;
}
internal void HandleRunAsyncUnexpectedFabricException(IServicePartition partition, FabricException fex)
{
ServiceTrace.Source.WriteErrorWithId(
this.traceType + ApiErrorTraceTypeSuffix,
this.traceId,
"RunAsync failed due to an unhandled FabricException causing replica to fault: {0}",
fex.ToString());
this.ReportRunAsyncUnexpectedExceptionHealth(partition, fex);
partition.ReportFault(FaultType.Transient);
}
public OrleansCommunicationListener(
StatelessServiceContext parameters,
ClusterConfiguration configuration,
IServicePartition servicePartition,
string deploymentId = null)
{
this.parameters = parameters;
this.configuration = configuration;
this.partition = servicePartition;
this.deploymentId = deploymentId;
}
public Task OpenAsync(IServicePartition partition, CancellationToken cancellationToken)
{
ActorTrace.Source.WriteInfoWithId(TraceType, this.traceId, "Opening ...");
this.ThrowIfClosed();
this.ArmGcTimer();
ActorTrace.Source.WriteInfoWithId(TraceType, this.traceId, "Opened.");
return(TaskDone.Done);
}
internal void HandleRunAsyncUnexpectedException(IServicePartition partition, Exception ex)
{
// ReSharper disable once UseStringInterpolation
var msg = string.Format(
"RunAsync failed due to an unhandled exception causing the host process to crash: {0}",
ex);
ServiceTrace.Source.WriteErrorWithId(this.traceType + ApiErrorTraceTypeSuffix, this.traceId, msg);
this.ReportRunAsyncUnexpectedExceptionHealth(partition, ex);
Environment.FailFast(msg);
}
public MockStatelessService(
StatelessServiceContext serviceContext,
IServiceEventSource serviceEventSource,
IServicePartition servicePartition)
{
this.serviceContext = serviceContext
?? throw new ArgumentNullException(nameof(serviceContext));
this.serviceEventSource = serviceEventSource
?? throw new ArgumentNullException(nameof(serviceEventSource));
this.servicePartition = servicePartition
?? throw new ArgumentNullException(nameof(servicePartition));
}
public ServiceMonitor(ServiceContext serviceContext, IServicePartition partition)
{
if (serviceContext == null)
{
throw new ArgumentNullException(nameof(serviceContext));
}
if (partition == null)
{
throw new ArgumentNullException(nameof(partition));
}
this.serviceContext = serviceContext;
this.partition = partition;
}
public void Start(IServicePartition partition, TimeSpan reportLoadInterval)
{
_partition = partition;
_reportInterval = reportLoadInterval;
foreach (var metric in _metrics)
{
metric.Start();
}
_timeIntervalStarted = DateTime.UtcNow;
_timer.Interval = Math.Min(1000, _reportInterval.TotalMilliseconds / 2);
_timer.Elapsed += OnTimerElapsed;
_timer.Start();
}
public ServiceMonitor(ServiceContext serviceContext, IServicePartition partition)
{
if (serviceContext == null)
{
throw new ArgumentNullException(nameof(serviceContext));
}
if (partition == null)
{
throw new ArgumentNullException(nameof(partition));
}
this.serviceContext = serviceContext;
this.partition = partition;
}
private void ReportPartitionHealth(IServicePartition partition, HealthInformation healthInformation)
{
try
{
partition.ReportPartitionHealth(healthInformation);
}
catch (Exception ex)
{
ServiceTrace.Source.WriteWarningWithId(
this.traceType,
this.traceId,
"ReportPartitionHealth() failed with: {0} while reporting health information: {1}.",
ex.ToString(),
healthInformation.ToString());
}
}
public MockStatefulService(
StatefulServiceContext serviceContext,
IServiceEventSource serviceEventSource,
IServicePartition servicePartition,
IReliableStateManager reliableStateManager)
{
this.serviceContext = serviceContext
?? throw new ArgumentNullException(nameof(serviceContext));
this.serviceEventSource = serviceEventSource
?? throw new ArgumentNullException(nameof(serviceEventSource));
this.servicePartition = servicePartition
?? throw new ArgumentNullException(nameof(servicePartition));
this.reliableStateManager = reliableStateManager;
}
/// <summary>
/// Initializes a new instance of the metrics type
/// </summary>
/// <param name="traceId">A unique identifier used to correlate the debugging and diagnostics messages</param>
/// <param name="logger">An instance used to write debugging and diagnostics information</param>
/// <param name="componentName">The name of the component for debugging and diagnostics messages</param>
/// <param name="context">The service fabric context that is assocaited with the entity being reported on</param>
/// <param name="partition">The partition of the service the metrics are being reported for</param>
/// <param name="metricsCallback">The method that is called to report metrics</param>
/// <param name="reportingInterval">How often the report will be sent</param>
public MetricReporter(Guid traceId, ILogger logger, string componentName, ServiceContext context, IServicePartition partition, GetMetrics metricsCallback, TimeSpan?reportingInterval = null)
{
this.logger = logger;
this.componentName = componentName;
this.logger.Informational(traceId, this.componentName, "Instantiated metrics reporter");
this.interval = reportingInterval ?? TimeSpan.FromSeconds(30);
if (this.interval < TimeSpan.FromSeconds(5))
{
this.interval = TimeSpan.FromSeconds(15);
}
this.metricsCallback = metricsCallback;
this.reportMetrics = context.CodePackageActivationContext.GetServiceTypes()[context.ServiceTypeName].LoadMetrics?.Select(item => item.Name).ToArray();
this.partition = partition;
}
internal void HandleRunAsyncUnexpectedException(IServicePartition partition, Exception ex)
{
// ReSharper disable once UseStringInterpolation
var msg = $"RunAsync failed due to an unhandled exception causing the host process to crash: {ex}";
ServiceTrace.Source.WriteErrorWithId(this.traceType + ApiErrorTraceTypeSuffix, this.traceId, msg);
this.ReportRunAsyncUnexpectedExceptionHealth(partition, ex);
// In LRC test we have observed that sometimes FailFast takes time to write error
// details to WER and bring down the service host. This causes delays in failover
// and availibility loss to service.
//
// Report fault transient and post FailFast on another thread to unblock ChangeRole.
// Service host will come down once FailFast completes.
//
partition.ReportFault(FaultType.Transient);
Task.Run(() => Environment.FailFast(msg));
}
internal async Task AwaitRunAsyncWithHealthReporting(IServicePartition partition, Task runAsyncTask)
{
while (true)
{
var delayTaskCts = new CancellationTokenSource();
var delayTask = Task.Delay(RunAsyncExpectedCancellationTimeSpan, delayTaskCts.Token);
var finishedTask = await Task.WhenAny(runAsyncTask, delayTask);
if (finishedTask == runAsyncTask)
{
delayTaskCts.Cancel();
ObserveExceptionIfAny(delayTask);
await runAsyncTask;
break;
}
var msg = $"RunAsync is taking longer then expected time ({RunAsyncExpectedCancellationTimeSpan.TotalSeconds}s) to cancel.";
ServiceTrace.Source.WriteWarningWithId(this.traceType + ApiSlowTraceTypeSuffix, this.traceId, msg);
this.ReportRunAsyncSlowCancellationHealth(partition, msg);
}
}
private void ReportRunAsyncUnexpectedExceptionHealth(IServicePartition partition, Exception unexpectedException)
{
var healthInfo = GetRunAsyncUnexpectedExceptionHealthInformation(unexpectedException);
this.ReportPartitionHealth(partition, healthInfo);
}
private void ReportRunAsyncSlowCancellationHealth(IServicePartition partition, string description)
{
var healthInfo = GetRunAsyncSlowCancellationHealthInformation(description);
this.ReportPartitionHealth(partition, healthInfo);
}
public Task OpenAsync(IServicePartition partition, CancellationToken cancellationToken)
{
return(TaskDone.Done);
}
public Task OpenAsync(IServicePartition servicePartition, CancellationToken cancellationToken)
{
return(this.ActorManager.OpenAsync(servicePartition, cancellationToken));
}
/// <summary>
/// Monitors the current silo, reporting a fault to the specified <paramref name="partition"/> if it faults.
/// </summary>
/// <param name="partition">
/// The partition.
/// </param>
private void MonitorSilo(IServicePartition partition)
{
this.fabricSilo.Stopped.ContinueWith(
_ =>
{
if (_.IsFaulted)
{
partition.ReportFault(FaultType.Transient);
this.stopped.TrySetException(
_.Exception ?? new Exception(typeof(OrleansFabricSilo).Name + " faulted."));
}
else if (_.IsCanceled)
{
this.stopped.TrySetCanceled();
}
else if (_.IsCompleted)
{
this.stopped.TrySetResult(0);
}
},
TaskContinuationOptions.ExecuteSynchronously);
}
/// <summary>
/// Initializes a new instance of the metrics type
/// </summary>
/// <param name="traceId">A unique identifier used to correlate the debugging and diagnostics messages</param>
/// <param name="logger">An instance used to write debugging and diagnostics information</param>
/// <param name="componentName">The name of the component for debugging and diagnostics messages</param>
/// <param name="context">The service fabric context that is assocaited with the entity being reported on</param>
/// <param name="partition">The partition of the service the metrics are being reported for</param>
/// <param name="metricsCallback">The method that is called to report metrics</param>
/// <param name="reportingInterval">How often the report will be sent</param>
public MetricReporter(Guid traceId, ILogger logger, string componentName, ServiceContext context, IServicePartition partition, GetMetrics metricsCallback, TimeSpan? reportingInterval = null)
{
this.logger = logger;
this.componentName = componentName;
this.logger.Informational(traceId, this.componentName, "Instantiated metrics reporter");
this.interval = reportingInterval ?? TimeSpan.FromSeconds(30);
if (this.interval < TimeSpan.FromSeconds(5)) this.interval = TimeSpan.FromSeconds(15);
this.metricsCallback = metricsCallback;
this.reportMetrics = context.CodePackageActivationContext.GetServiceTypes()[context.ServiceTypeName].LoadMetrics?.Select(item => item.Name).ToArray();
this.partition = partition;
}
