/// <summary>
/// Log cluster options on startup
/// </summary>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="nodeCount">Number of nodes</param>
protected void LogClusterOptions(CircuitBreakerOptions circuitBreakerOptions, int nodeCount)
{
var version = Helper.GetVersion();
Logger.LogInformation($@"
Initializing cluster v{version}...
# Number of nodes: {nodeCount}
{(ClusterOptions.ExecuteRemotely ? $"# Executing remotely with configured hosts: {Environment.NewLine}{string.Join(Environment.NewLine, ClusterOptions.Hosts.Select(host => $"{host.MachineName}:{host.Port}"))}" : "# Executing locally")}
# Processing type: {Enum.GetName(typeof(ClusterProcessingType), ClusterOptions.ClusterProcessingType)}
# Queuing type: {Enum.GetName(typeof(NodeQueuingStrategy), ClusterOptions.NodeQueuingStrategy)}
# Node throttling: {ClusterOptions.NodeThrottling * ClusterOptions.Window.TotalMilliseconds / 1000} op/s
# Limiting CPU Usage: {ClusterOptions.LimitCpuUsage} %
# Number of retry attempts on failing items: {ClusterOptions.RetryAttempt}
# Evict items on throttling: {ClusterOptions.EvictItemsWhenNodesAreFull}
{(ClusterOptions.PersistenceEnabled ? $"# Persistence enabled with maximum items in memory: {ClusterOptions.MaxItemsInPersistentCache}" : "# Persistence disabled")}
");
Logger.LogInformation($@"
Setting cluster circuit breaker options...
# Duration of break: {circuitBreakerOptions.CircuitBreakerDurationOfBreak:g}
# Failure threshold: {circuitBreakerOptions.CircuitBreakerFailureThreshold}
# Minimum throughput: {circuitBreakerOptions.CircuitBreakerMinimumThroughput}
# Sampling duration: {circuitBreakerOptions.CircuitBreakerSamplingDuration}
");
}
/// <summary>
/// <see cref="AsyncSequentialDispatcherLocalNode{TInput,TOutput}"/>
/// </summary>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public AsyncSequentialDispatcherLocalNode(
IProgress <double> progress,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_clusterOptions = clusterOptions;
NodeMetrics = new NodeMetrics(Guid.NewGuid());
}
/// <summary>
/// Constructor
/// </summary>
public CircuitBreaker(CircuitBreakerOptions options)
{
InternalContract.RequireNotNull(options, nameof(options));
InternalContract.RequireValidated(options, nameof(options));
_options = options;
_options.CoolDownStrategy.Reset();
_commonCancellationTokenSource = new CancellationTokenSource();
}
/// <summary>
/// Allows <see cref="CircuitBreaker"/> configuration
/// </summary>
/// <param name="builder">The <see cref="CircuitBreakerOptionsBuilder"/></param>
/// <returns><see cref="RecoveryOptionsBuilder"/></returns>
public RecoveryOptionsBuilder WithCircuitBreaker(Action <CircuitBreakerOptionsBuilder> builder)
{
var b = CircuitBreakerOptionsBuilder.Create(_services);
builder?.Invoke(b);
_circuitBreakerOptions = b.Build();
return(this);
}
public static Options UseCircuitBreaker(this Options optionsBuilder, CircuitBreakerOptions circuitBreakerOptions = null)
=> optionsBuilder
.UseMessageBehaviour <CircuitBreakerMessageProcessingBehaviour>()
.UseBatchBehaviour <CircuitBreakerBatchProcessingBehaviour>()
.UseServices(s => s
.AddSingleton(circuitBreakerOptions ?? new CircuitBreakerOptions())
.AddSingleton <CircuitBreakerMessageProcessingBehaviour>()
.AddSingleton <CircuitBreakerBatchProcessingBehaviour>());
public void SetDefaultValues()
{
var options = new CircuitBreakerOptions();
Assert.True(options.InitialState == CircuitState.Closed);
Assert.Equal(TimeSpan.FromSeconds(30), options.DefaultCooldownPeriod);
Assert.Equal(1, options.HalfOpenSuccessCountBeforeClose);
}
/// <summary>
/// <see cref="UnarySequentialDispatcherRemoteNode{TInput}"/>
/// </summary>
/// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="host"><see cref="Host"/></param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public UnarySequentialDispatcherRemoteNode(
IAppCache persistentCache,
IProgress <double> progress,
Host host,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_clusterOptions = clusterOptions;
ISubject <PersistentItem <TInput> > dispatcherSubject = new Subject <PersistentItem <TInput> >();
_synchronizedDispatcherSubject = Subject.Synchronize(dispatcherSubject);
_synchronizedDispatcherSubjectSubscription = _synchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItemAsync(item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
var channel = new Channel(host.MachineName, host.Port,
ChannelCredentials.Insecure);
_remoteContract = MagicOnionClient.Create <IRemoteContract <TInput> >(channel);
IRemoteNodeSubject nodeReceiver = new NodeReceiver(_logger);
_remoteNodeHealthSubscription =
nodeReceiver.RemoteNodeHealthSubject.Subscribe(remoteNodeHealth =>
{
NodeMetrics.RemoteNodeHealth = remoteNodeHealth;
});
_nodeHub = StreamingHubClient.Connect <INodeHub, INodeReceiver>(channel, (INodeReceiver)nodeReceiver);
NodeMetrics = new NodeMetrics(Guid.NewGuid());
}
public void Equals_Both_Null_Returns_True()
{
// Arrange
// Act
var equals = CircuitBreakerOptions.Equals(null, null);
// Assert
Assert.True(equals);
}
public static void SetupCircuitBreaker(this IServiceCollection services, CircuitBreakerOptions circuitBreakerOptions)
{
if (circuitBreakerOptions?.Endpoints == null || !circuitBreakerOptions.Endpoints.Any())
{
return;
}
CheckCertificate = circuitBreakerOptions.CheckCertificate;
services.AddHttpClient(circuitBreakerOptions.Endpoints);
services.AddTransient <ICircuitBreakerHttpClient, CircuitBreakerHttpClient>();
}
public Luffy UseCircuitBreaker(CircuitBreakerOptions circuitBreakerOptions, ICircuitBreakerStateStore stateStore = null)
{
_circuitBreakerOptions = circuitBreakerOptions;
if (stateStore != null)
{
_circuitBreakerStateStore = stateStore;
}
return(this);
}
public void Equals_Second_Null_Returns_False()
{
var options1 = new CircuitBreakerOptions
{
MaxConcurrentRequests = 10,
MaxConcurrentRetries = 5,
};
var equals = CircuitBreakerOptions.Equals(options1, null);
Assert.False(equals);
}
public void Equals_First_Null_Returns_False()
{
var options2 = new CircuitBreakerOptions
{
MaxConcurrentRequests = 20,
MaxConcurrentRetries = 10,
};
var equals = CircuitBreakerOptions.Equals(null, options2);
Assert.False(equals);
}
public void DeepClone_Works()
{
var sut = new CircuitBreakerOptions
{
MaxConcurrentRequests = 10,
MaxConcurrentRetries = 5,
};
var clone = sut.DeepClone();
Assert.NotSame(sut, clone);
Assert.Equal(sut.MaxConcurrentRequests, clone.MaxConcurrentRequests);
Assert.Equal(sut.MaxConcurrentRetries, clone.MaxConcurrentRetries);
}
internal static bool Equals(CircuitBreakerOptions options1, CircuitBreakerOptions options2)
{
if (options1 == null && options2 == null)
{
return(true);
}
if (options1 == null || options2 == null)
{
return(false);
}
return(options1.MaxConcurrentRequests == options2.MaxConcurrentRequests &&
options1.MaxConcurrentRetries == options2.MaxConcurrentRetries);
}
/// <summary>
/// <see cref="UnarySequentialDispatcherLocalNode{TInput}"/>
/// </summary>
/// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
/// <param name="process">The <see cref="Task"/> to be applied to an item</param>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public UnarySequentialDispatcherLocalNode(
IAppCache persistentCache,
Func <TInput, NodeMetrics, CancellationToken, Task> process,
IProgress <double> progress,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_process = process;
_clusterOptions = clusterOptions;
ISubject <PersistentItem <TInput> > dispatcherSubject = new Subject <PersistentItem <TInput> >();
_synchronizedDispatcherSubject = Subject.Synchronize(dispatcherSubject);
_synchronizedDispatcherSubjectSubscription = _synchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItemAsync(item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
NodeMetrics = new NodeMetrics(Guid.NewGuid());
}
public void DeepClone_Works()
{
// Arrange
var sut = new CircuitBreakerOptions
{
MaxConcurrentRequests = 10,
MaxConcurrentRetries = 5,
};
// Act
var clone = sut.DeepClone();
// Assert
clone.Should().NotBeSameAs(sut);
clone.MaxConcurrentRequests.Should().Be(sut.MaxConcurrentRequests);
clone.MaxConcurrentRetries.Should().Be(sut.MaxConcurrentRetries);
}
public void Equals_Different_Value_Returns_False()
{
var options1 = new CircuitBreakerOptions
{
MaxConcurrentRequests = 10,
MaxConcurrentRetries = 5,
};
var options2 = new CircuitBreakerOptions
{
MaxConcurrentRequests = 20,
MaxConcurrentRetries = 10,
};
var equals = CircuitBreakerOptions.Equals(options1, options2);
Assert.False(equals);
}
public void ShouldReturnSameBreakerWhenCreateMultiple()
{
// given
var optionMock = new Mock <IOptions <CircuitBreakerOptions> >();
var opt = new CircuitBreakerOptions();
optionMock.Setup(x => x.Value).Returns(opt);
var factory = new CircuitBreakerFactory(optionMock.Object);
// when
var cb = factory.CreateCircuitBreaker("test");
var cb2 = factory.CreateCircuitBreaker("test");
// then
cb.Should().BeEquivalentTo(cb2);
}
/// <summary>
/// <see cref="AsyncDispatcherRemoteNode{TInput,TOutput}"/>
/// </summary>
/// <param name="host"><see cref="Host"/></param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public AsyncDispatcherRemoteNode(
Host host,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, cts, logger)
{
_logger = logger;
_clusterOptions = clusterOptions;
_channel = new Channel(host.MachineName, host.Port,
ChannelCredentials.Insecure);
_remoteContract = MagicOnionClient.Create <IOutputRemoteContract <TInput, TOutput> >(_channel);
IRemoteNodeSubject nodeReceiver = new NodeReceiver(_logger);
_remoteNodeHealthSubscription =
nodeReceiver.RemoteNodeHealthSubject.Subscribe(remoteNodeHealth =>
{
NodeMetrics.RemoteNodeHealth = remoteNodeHealth;
});
_nodeHub = StreamingHubClient.Connect <INodeHub, INodeReceiver>(_channel, (INodeReceiver)nodeReceiver);
NodeMetrics = new NodeMetrics(Guid.NewGuid());
}
/// <summary>
/// Create the Submit Pipeline. The submit pipeline can be customized
/// with custom submit pipeline elements to control the submission
/// behaviour. The submit pipeline is stateless, so there is only one
/// instance of each pipeline that is shared across all submissions.
/// </summary>
private void CreateSubmitPipeline()
{
var apiClientFactory = new ApiClientFactory();
var httpSubmitItemElement = new HttpSubmitItemPipelineElement(null);
httpSubmitItemElement.ApiClientFactory = apiClientFactory;
// Create a Filter pipeline element for items. This implements the logic
// to apply the user-defined filtering on item submission.
// Pass the httpSubmitItemElement into the FilterPipelineElement to create a chain of pipeline elements.
// The filter is called first, followed by the submission element.
var itemFilterPipelineElement = new FilterPipelineElement(httpSubmitItemElement);
_itemSubmitPipeline = itemFilterPipelineElement;
var httpSubmitAggregationElement = new HttpSubmitAggregationPipelineElement(null);
httpSubmitAggregationElement.ApiClientFactory = apiClientFactory;
var aggregationFilterPipelineElement = new FilterPipelineElement(httpSubmitAggregationElement);
_aggregationSubmitPipeline = aggregationFilterPipelineElement;
var circuitBreakerOpts = new CircuitBreakerOptions();
var httpSubmitBinaryElement = new DirectSubmitBinaryPipelineElement(null)
{
ApiClientFactory = apiClientFactory,
CircuitProvider = new AzureBlobRetryProviderWithCircuitBreaker(circuitBreakerOpts, true),
RetryProvider = new AzureBlobRetryProviderWithCircuitBreaker(circuitBreakerOpts, true),
// Log = optional logger
};
var binaryFilterPipelineElement = new FilterPipelineElement(httpSubmitBinaryElement);
_binarySubmitPipeline = binaryFilterPipelineElement;
var httpSubmitAuditEventElement = new HttpSubmitAuditEventPipelineElement(null);
httpSubmitAuditEventElement.ApiClientFactory = apiClientFactory;
_auditEventSubmitPipeline = httpSubmitAuditEventElement;
}
public void Equals_Same_Value_Returns_True()
{
// Arrange
var options1 = new CircuitBreakerOptions
{
MaxConcurrentRequests = 10,
MaxConcurrentRetries = 5,
};
var options2 = new CircuitBreakerOptions
{
MaxConcurrentRequests = 10,
MaxConcurrentRetries = 5,
};
// Act
var equals = CircuitBreakerOptions.Equals(options1, options2);
// Assert
Assert.True(equals);
}
private void InitializeCircuitBreaker(IHttpClientHandler httpClientHandler,
IOptions <CircuitBreakerOptions> circuitBreakerOptions)
{
_httpClientHandler = httpClientHandler;
_circuitBreakerOptions = circuitBreakerOptions.Value;
}
/// <summary>
/// <see cref="DualSequentialDispatcherLocalNode{TInput1,TInput2,TOutput1,TOutput2}"/>
/// </summary>
/// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
/// <param name="item1Resolver">The <see cref="Task"/> to be applied to an item</param>
/// <param name="item2Resolver">The <see cref="Task"/> to be applied to an item</param>
/// <param name="resolver">The <see cref="Task"/> to be applied to an item</param>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public DualSequentialDispatcherLocalNode(
IAppCache persistentCache,
Func <TInput1, NodeMetrics, CancellationToken, Task <TOutput1> > item1Resolver,
Func <TInput2, NodeMetrics, CancellationToken, Task <TOutput2> > item2Resolver,
Func <TOutput1, TOutput2, NodeMetrics, CancellationToken, Task> resolver,
IProgress <double> progress,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_item1Resolver = item1Resolver;
_item2Resolver = item2Resolver;
_clusterOptions = clusterOptions;
ISubject <LinkedItem <TInput1> > item1DispatcherSubject = new Subject <LinkedItem <TInput1> >();
_item1SynchronizedDispatcherSubject = Subject.Synchronize(item1DispatcherSubject);
_item1SynchronizedDispatcherSubjectSubscription = _item1SynchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItem1Async(item.Key.ToString(), item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
ISubject <LinkedItem <TInput2> > item2DispatcherSubject = new Subject <LinkedItem <TInput2> >();
_item2SynchronizedDispatcherSubject = Subject.Synchronize(item2DispatcherSubject);
_item2SynchronizedDispatcherSubjectSubscription = _item2SynchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItem2Async(item.Key.ToString(), item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
NodeMetrics = new NodeMetrics(Guid.NewGuid());
var item1ProcessSource = new ConcurrentDictionary <Guid, TOutput1>();
var item2ProcessSource = new ConcurrentDictionary <Guid, TOutput2>();
var joinBlock =
new JoinBlock <KeyValuePair <Guid, CancellationTokenSource>, KeyValuePair <Guid, CancellationTokenSource> >(
new GroupingDataflowBlockOptions {
Greedy = false
});
_item1Source =
new TransformBlock <Tuple <Guid, TOutput1, CancellationTokenSource>,
KeyValuePair <Guid, CancellationTokenSource>
>(source =>
{
if (!item1ProcessSource.ContainsKey(source.Item1) &&
!item1ProcessSource.TryAdd(source.Item1, source.Item2))
{
_logger.LogError(
$"Could not add item of type {source.Item2.GetType()} and key {source.Item1.ToString()} to the buffer.");
}
return(new KeyValuePair <Guid, CancellationTokenSource>(source.Item1, source.Item3));
});
_item2Source =
new TransformBlock <Tuple <Guid, TOutput2, CancellationTokenSource>,
KeyValuePair <Guid, CancellationTokenSource>
>(
source =>
{
if (!item2ProcessSource.ContainsKey(source.Item1) &&
!item2ProcessSource.TryAdd(source.Item1, source.Item2))
{
_logger.LogError(
$"Could not add item of type {source.Item2.GetType()} and key {source.Item1.ToString()} to the buffer.");
}
return(new KeyValuePair <Guid, CancellationTokenSource>(source.Item1, source.Item3));
});
var processBlock = new ActionBlock <Tuple <KeyValuePair <Guid, CancellationTokenSource>,
KeyValuePair <Guid, CancellationTokenSource> > >(
async combined =>
{
var policy = Policy
.Handle <Exception>(ex => !(ex is TaskCanceledException || ex is OperationCanceledException))
.WaitAndRetryAsync(_clusterOptions.RetryAttempt,
retryAttempt =>
TimeSpan.FromSeconds(Math.Pow(10, retryAttempt)),
(exception, sleepDuration, retry, context) =>
{
if (retry >= _clusterOptions.RetryAttempt)
{
_logger.LogError(
$"Could not process item after {retry} retry times: {exception.Message}");
}
});
var policyResult = await policy.ExecuteAndCaptureAsync(async ct =>
{
try
{
if (CpuUsage > _clusterOptions.LimitCpuUsage)
{
var suspensionTime = (CpuUsage - _clusterOptions.LimitCpuUsage) / CpuUsage * 100;
await Task.Delay((int)suspensionTime, ct);
}
if (item1ProcessSource.ContainsKey(combined.Item1.Key) &&
item2ProcessSource.ContainsKey(combined.Item2.Key) &&
item1ProcessSource.TryGetValue(combined.Item1.Key, out var item1) &&
item2ProcessSource.TryGetValue(combined.Item2.Key, out var item2))
{
await resolver(item1, item2, NodeMetrics, ct);
combined.Item1.Value.Cancel();
combined.Item2.Value.Cancel();
}
}
catch (Exception ex) when(ex is TaskCanceledException || ex is OperationCanceledException)
{
_logger.LogTrace("The item process has been cancelled.");
}
}, cts.Token).ConfigureAwait(false);
if (policyResult.Outcome == OutcomeType.Failure)
{
_logger.LogCritical(
policyResult.FinalException != null
? $"Could not process item: {policyResult.FinalException.Message}."
: "An error has occured while processing the item.");
}
if (!item1ProcessSource.TryRemove(combined.Item1.Key, out _))
{
_logger.LogWarning(
$"Could not remove item of key {combined.Item1.ToString()} from the buffer.");
}
if (!item2ProcessSource.TryRemove(combined.Item2.Key, out _))
{
_logger.LogWarning(
$"Could not remove item of key {combined.Item2.ToString()} from the buffer.");
}
});
var options = new DataflowLinkOptions
{
PropagateCompletion = true
};
_item1Source.LinkTo(joinBlock.Target1, options);
_item2Source.LinkTo(joinBlock.Target2, options);
joinBlock.LinkTo(processBlock, options);
}
public CircuitBreakerBatchProcessingBehaviour(ILogger <CircuitBreakerBatchProcessingBehaviour> log, SubscriberConfiguration config, CircuitBreakerOptions circuitBreakerOptions)
{
_log = log;
_config = config;
ShouldCircuitBreak = circuitBreakerOptions.ShouldCircuitBreak;
_circuitTestIntervalInSeconds = circuitBreakerOptions.CircuitTestIntervalInSeconds;
State = CircuitState.Closed;
CircuitBroken += circuitBreakerOptions.CircuitBroken;
CircuitReset += circuitBreakerOptions.CircuitReset;
CircuitTest += circuitBreakerOptions.CircuitTest;
}
public void Equals_Both_Null_Returns_True()
{
var equals = CircuitBreakerOptions.Equals(null, null);
Assert.True(equals);
}
