private bool IsTransient(ITaskContext task, Exception ex)
{
var isTrans = TransientFaultHandling.IsTransient(ex);
if (!isTrans)
{
task.Logger.Warn($"Exception type {ex.GetType()} is not Transient. Msg {ex.Message}");
}
return(isTrans);
}
/// <summary>
/// Constructs request for deploying SMM storage objects to target GSM database.
/// </summary>
/// <param name="credentials">Credentials for connection.</param>
/// <param name="retryPolicy">Retry policy.</param>
/// <param name="operationName">Operation name, useful for diagnostics.</param>
/// <param name="createMode">Creation mode.</param>
/// <param name="targetVersion">target version of store to deploy</param>
internal CreateShardMapManagerGlobalOperation(
SqlShardMapManagerCredentials credentials,
TransientFaultHandling.RetryPolicy retryPolicy,
string operationName,
ShardMapManagerCreateMode createMode,
Version targetVersion) :
base(credentials, retryPolicy, operationName)
{
_createMode = createMode;
_targetVersion = targetVersion;
}
/// <summary>
/// Constructs an instance of SqlOperationLocal.
/// </summary>
/// <param name="credentials">Credentials for connecting to SMM databases.</param>
/// <param name="retryPolicy">Retry policy for requests.</param>
/// <param name="location">Shard location where the operation is to be performed.</param>
/// <param name="operationName">Operation name.</param>
internal StoreOperationLocal(
SqlShardMapManagerCredentials credentials,
TransientFaultHandling.RetryPolicy retryPolicy,
ShardLocation location,
string operationName)
{
_credentials = credentials;
_retryPolicy = retryPolicy;
this.OperationName = operationName;
this.Location = location;
}
public void Start()
{
this.log.Log($"Starting mail sender...");
var outbox = TransientFaultHandling.RetryIfFail(TimeSpan.FromDays(1), () => this.repo.Get <Outbox>(this.outboxStreamName));
if (outbox == null)
{
outbox = new Outbox(this.outboxStreamName, DateTime.Now);
}
this.sub = this.connection.SubscribeToStreamFrom(this.queueStreamName, outbox.CurrentPosition, CatchUpSubscriptionSettings.Default,
(s, rawEvent) =>
{
var e = this.serializer.Deserialize(rawEvent) as NewOutgoingEmail;
if (e != null)
{
var env = e.Envelope;
if (env == null)
{
this.log.Trace($"Ignoring null envelope in event #{rawEvent.OriginalEventNumber}");
}
else
{
outbox.SendEmail(rawEvent.OriginalEventNumber, e.EmailId, e.Envelope, this.sender);
this.repo.Save(outbox);
}
}
else
{
this.log.Trace($"Ignoring event #{rawEvent.OriginalEventNumber} {rawEvent.OriginalEvent.EventType}");
}
},
s => this.log.Log($"The mail sending is now processing in real time..."),
(s, r, e) =>
{
if (r == SubscriptionDropReason.ConnectionClosed)
{
this.log.Error(e, $"The subscription to {s.StreamId} was droped. Reason: {r}. Waiting for new connection...");
return;
}
if (r == SubscriptionDropReason.CatchUpError)
{
this.log.Error(e, $"The subscription to {s.StreamId} was droped. Reason: {r}");
Thread.Sleep(1000);
this.Start();
return;
}
throw e;
});
}
/// <summary>
/// Constructs request for obtaining shard map manager object if the GSM has the SMM objects in it.
/// </summary>
/// <param name="credentials">Credentials for connection.</param>
/// <param name="retryPolicy">Retry policy.</param>
/// <param name="operationName">Operation name, useful for diagnostics.</param>
/// <param name="throwOnFailure">Whether to throw exception on failure or return error code.</param>
/// <returns>The store operation.</returns>
public virtual IStoreOperationGlobal CreateGetShardMapManagerGlobalOperation(
SqlShardMapManagerCredentials credentials,
TransientFaultHandling.RetryPolicy retryPolicy,
string operationName,
bool throwOnFailure)
{
return new GetShardMapManagerGlobalOperation(
credentials,
retryPolicy,
operationName,
throwOnFailure);
}
/// <summary>
/// Constructs request for deploying SMM storage objects to target GSM database.
/// </summary>
/// <param name="credentials">Credentials for connection.</param>
/// <param name="retryPolicy">Retry policy.</param>
/// <param name="operationName">Operation name, useful for diagnostics.</param>
/// <param name="createMode">Creation mode.</param>
/// <param name="targetVersion">target version of store to deploy, this will be used mainly for upgrade testing purposes.</param>
/// <returns>The store operation.</returns>
public virtual IStoreOperationGlobal CreateCreateShardMapManagerGlobalOperation(
SqlShardMapManagerCredentials credentials,
TransientFaultHandling.RetryPolicy retryPolicy,
string operationName,
ShardMapManagerCreateMode createMode,
Version targetVersion)
{
return new CreateShardMapManagerGlobalOperation(
credentials,
retryPolicy,
operationName,
createMode,
targetVersion);
}
public static void Initialize(string connectionString)
{
TransientFaultHandling.InitializeRetryManager();
// Setup the database
DbUp.DeployChanges.To
.SqlDatabase(connectionString)
.WithScriptsEmbeddedInAssemblies(new[] {
typeof(RelationalStore).Assembly, // Contains the Nevermore required script
typeof(StorageInitializer).Assembly // Your scripts
})
.WithScript("schema-startup", CreateStartupSchema(connectionString))
.Build()
.PerformUpgrade();
}
private bool TrackException(Exception ex, bool isRetry = false)
{
// For simplicity in this example, open the circuit breaker on the first exception.
// In reality this would be more complex. A certain type of exception, such as one
// that indicates a service is offline, might trip the circuit breaker immediately.
// Alternatively it may count exceptions locally or across multiple instances and
// use this value over time, or the exception/success ratio based on the exception
// types, to open the circuit breaker.
var isTransient = TransientFaultHandling.IsCircuitBreaker(ex);
if (isTransient && isRetry)
{
_logger.Warn("Circuit breaker going to trip due to error {exception}", ex);
this.stateStore.Trip(ex);
}
return(isTransient);
}
static IntegrationTestDatabase()
{
TransientFaultHandling.InitializeRetryManager();
}
//public override TU FeatureDecoratorQuery(T message)
//{
// var task = RetryExecuteQuery(message);
// return task.Result;
//}
void RetryExecute(T message)
{
int currentRetry = 0;
//IRetryableMessage<T> retryable=new RetryableMessage<T>(message){IsRetry = false};
for (;;)
{
try
{
_token.ThrowIfCancellationRequested();
_breaker.Invoke(
// This is the operation we want to execute.
() => Handler?.Handle(message), currentRetry > 0
);
// Return or break.
break;
}
catch (CircuitBreakerOpenException e)
{
_token.ThrowIfCancellationRequested();
_logger.Warn($"CircuitBreaker '{_name}' Open with internal exception {e.InnerException?.Message}");
//Thread.Sleep(Math.Max(_delayInRetries, 100));
//await Task.Delay(Math.Min(_delayInRetries, 100));
Task.Delay(Math.Min(_delayInRetries, 100)).Wait(_token);
//ignore and retry. this is not a genuine error
}
catch (AggregateException aex) when(aex.InnerExceptions.Any(r => r is FrameworkException))
{
var fex = aex.InnerExceptions.First(r => r is FrameworkException);
//if (fex != null)
{
ExceptionDispatchInfo.Capture(fex).Throw();
}
}
catch (Exception ex) when(!(ex is FrameworkException))
{
_token.ThrowIfCancellationRequested();
_logger.Error($"Operation Exception in Retry Handler {ex.Message}");
Robustness.Instance.SafeCall(() => _errorAction?.Invoke(ex), _logger, "Error invoking error action in RetryHandler {0}");
if (_maxRetries.HasValue) //infinite
{
currentRetry++;
}
// Check if the exception thrown was a transient exception
// based on the logic in the error detection strategy.
// Determine whether to retry the operation, as well as how
// long to wait, based on the retry strategy.
if (_maxRetries.HasValue && ((_maxRetries > 0 && currentRetry > _maxRetries) || !TransientFaultHandling.IsTransient(ex)))
{
// If this isn't a transient error or we shouldn't retry,
// rethrow the exception.
throw;
}
}
// Wait to retry the operation.
// Consider calculating an exponential delay here and
// using a strategy best suited for the operation and fault.
try
{
Task.Delay(_delayInRetries, _token).Wait(_token);
}
catch (TaskCanceledException)
{
//cancelled
}
}
}
//public override TU FeatureDecoratorQuery(T message)
//{
// var task = RetryExecuteQuery(message);
// return task.Result;
//}
void RetryExecute(T message)
{
int currentRetry = 0;
//IRetryableMessage<T> retryable=new RetryableMessage<T>(message){IsRetry = false};
for (;;)
{
try
{
_token.ThrowIfCancellationRequested();
Handler?.Handle(message);
//await TransientOperationAsync();
// Return or break.
break;
}
catch (Exception ex) when(!(ex is FrameworkException))
{
_token.ThrowIfCancellationRequested();
_logger.Error($"Operation Exception in Retry Handler {ex.Message}");
Robustness.Instance.SafeCall(() => _errorAction?.Invoke(ex), _logger, "Error invoking error action in RetryHandler {0}");
if (_maxRetries.HasValue) //infinite
{
currentRetry++;
}
// Check if the exception thrown was a transient exception
// based on the logic in the error detection strategy.
// Determine whether to retry the operation, as well as how
// long to wait, based on the retry strategy.
if (_maxRetries.HasValue && ((_maxRetries > 0 && currentRetry > _maxRetries) || !TransientFaultHandling.IsTransient(ex)))
{
// If this isn't a transient error or we shouldn't retry,
// rethrow the exception.
throw;
}
}
// Wait to retry the operation.
// Consider calculating an exponential delay here and
// using a strategy best suited for the operation and fault.
try
{
Task.Delay(_delayInRetries, _token).Wait(_token);
}
catch (TaskCanceledException)
{
//cancelled
}
}
}
/// <summary>
/// Constructs request for obtaining shard map manager object if the GSM has the SMM objects in it.
/// </summary>
/// <param name="credentials">Credentials for connection.</param>
/// <param name="retryPolicy">Retry policy.</param>
/// <param name="operationName">Operation name, useful for diagnostics.</param>
/// <param name="throwOnFailure">Whether to throw exception on failure or return error code.</param>
internal GetShardMapManagerGlobalOperation(SqlShardMapManagerCredentials credentials, TransientFaultHandling.RetryPolicy retryPolicy, string operationName, bool throwOnFailure) :
base(credentials, retryPolicy, operationName)
{
_throwOnFailure = throwOnFailure;
}
/// <summary>
/// Sets the stub of StoreOperationFactory.CreateGetShardMapManagerGlobalOperation(SqlShardMapManagerCredentials credentials, RetryPolicy retryPolicy, String operationName, Boolean throwOnFailure)
/// </summary>
public override IStoreOperationGlobal CreateGetShardMapManagerGlobalOperation(SqlShardMapManagerCredentials credentials, TransientFaultHandling.RetryPolicy retryPolicy, string operationName, bool throwOnFailure)
{
Func<SqlShardMapManagerCredentials, TransientFaultHandling.RetryPolicy, string, bool, IStoreOperationGlobal> func1 = this.CreateGetShardMapManagerGlobalOperationSqlShardMapManagerCredentialsTransientFaultHandlingRetryPolicyStringBoolean;
if (func1 != null)
return func1(credentials, retryPolicy, operationName, throwOnFailure);
if (this.___callBase)
return base.CreateGetShardMapManagerGlobalOperation(credentials, retryPolicy, operationName, throwOnFailure);
return this.InstanceBehavior.Result<StubStoreOperationFactory, IStoreOperationGlobal>(this, "CreateGetShardMapManagerGlobalOperation");
}
/// <summary>
/// Sets the stub of StoreOperationFactory.CreateCreateShardMapManagerGlobalOperation(SqlShardMapManagerCredentials credentials, RetryPolicy retryPolicy, String operationName, ShardMapManagerCreateMode createMode, Version targetVersion)
/// </summary>
public override IStoreOperationGlobal CreateCreateShardMapManagerGlobalOperation(SqlShardMapManagerCredentials credentials, TransientFaultHandling.RetryPolicy retryPolicy, string operationName, ShardMapManagerCreateMode createMode, Version targetVersion)
{
Func<SqlShardMapManagerCredentials, TransientFaultHandling.RetryPolicy, string, ShardMapManagerCreateMode, Version, IStoreOperationGlobal> func1 = this.CreateCreateShardMapManagerGlobalOperationSqlShardMapManagerCredentialsTransientFaultHandlingRetryPolicyStringShardMapManagerCreateModeVersion;
if (func1 != null)
return func1(credentials, retryPolicy, operationName, createMode, targetVersion);
if (this.___callBase)
return base.CreateCreateShardMapManagerGlobalOperation(credentials, retryPolicy, operationName, createMode, targetVersion);
return this.InstanceBehavior.Result<StubStoreOperationFactory, IStoreOperationGlobal>(this, "CreateCreateShardMapManagerGlobalOperation");
}
/// <summary>
/// Initializes new instance of <see cref="RetryingEventArgs"/> class.
/// </summary>
/// <param name="arg">RetryingEventArgs from RetryPolicy.Retrying event.</param>
internal RetryingEventArgs(TransientFaultHandling.RetryingEventArgs arg)
{
this.CurrentRetryCount = arg.CurrentRetryCount;
this.Delay = arg.Delay;
this.LastException = arg.LastException;
}
/// <summary>
/// Constructs an instance of SqlOperationGlobal.
/// </summary>
/// <param name="credentials">Credentials for connecting to SMM GSM database.</param>
/// <param name="retryPolicy">Retry policy for requests.</param>
/// <param name="operationName">Operation name, useful for diagnostics.</param>
internal StoreOperationGlobal(SqlShardMapManagerCredentials credentials, TransientFaultHandling.RetryPolicy retryPolicy, string operationName)
{
this.OperationName = operationName;
_credentials = credentials;
_retryPolicy = retryPolicy;
}
