public async Task VerifyExceptionWhenCallHttpAsyncReturnsNonStatusOK()
{
FailureRequest mockFailureRequest = new FailureRequest
{
FailureTime = DateTime.UtcNow,
InstanceId = "TEST-INSTANCEID",
RequestId = "TEST-REQUESTID",
ResourceId = "TEST-RESOURCEID",
};
this.durableOrchestrationContextMock.Setup(client => client.GetInput <FailureRequest>()).Returns(mockFailureRequest);
this.durableOrchestrationContextMock.Setup(client => client.CallHttpAsync(It.IsAny <DurableHttpRequest>())).Returns(Task.FromResult(new DurableHttpResponse(HttpStatusCode.BadRequest)));
OpenCircuitOrchestrator function = new OpenCircuitOrchestrator();
Func <Task> testCode = async() => await function.OpenCircuit(this.durableOrchestrationContextMock.Object, this.logger).ConfigureAwait(false);
var ex = await Record.ExceptionAsync(testCode).ConfigureAwait(false);
Assert.NotNull(ex);
Assert.IsType <ApplicationException>(ex);
string expectedLog = "Failed to stop Function App";
Assert.Contains(expectedLog, ex.Message, StringComparison.OrdinalIgnoreCase);
}
public async Task AddFailureUponCrossingThreadholdShouldCallOpenCircuitOrchestrator()
{
Mock <IDurableClient> clientMock = new Mock <IDurableClient>();
clientMock.Setup(service => service.StartNewAsync("OpenCircuit", It.IsAny <FailureRequest>())).Returns(Task.FromResult(string.Empty)).Verifiable();
CircuitBreakerActor function = new CircuitBreakerActor(clientMock.Object, this.logger);
FailureRequest req = new FailureRequest
{
RequestId = "RequestId",
FailureTime = DateTime.UtcNow,
InstanceId = "InstanceId",
ResourceId = "ResourceID",
};
for (int i = 1; i <= 2; i++)
{
req.RequestId = $"Req{i}";
req.FailureTime = req.FailureTime.AddSeconds(10);
await function.AddFailure(req).ConfigureAwait(false);
}
string expectedLogText = "Break this circuit for entity";
Assert.Contains(expectedLogText, this.logger.Logs[1], StringComparison.OrdinalIgnoreCase);
Mock.Verify();
}
public async Task AddFailureFunctionShouldCallCircuitBreakerActorEntity()
{
FailureRequest failureReq = new FailureRequest
{
RequestId = "RequestId",
FailureTime = DateTime.UtcNow,
InstanceId = "InstanceId",
ResourceId = "ResourceId",
};
string entityKey = "AddFailureTest";
var reqData = JsonConvert.SerializeObject(failureReq);
using HttpRequestMessage req = new HttpRequestMessage
{
Content = new StringContent(reqData, Encoding.UTF8, "application/json"),
};
EntityId actualEntityId;
this.durableClientMock
.Setup(service => service.SignalEntityAsync <ICircuitBreakerActor>(It.IsAny <EntityId>(), It.IsAny <Action <ICircuitBreakerActor> >()))
.Callback <EntityId, Action <ICircuitBreakerActor> >((entityId, operation) => actualEntityId = entityId)
.Returns(Task.FromResult(string.Empty));
await CircuitBreakerFunction.AddFailureFunction(req, this.durableClientMock.Object, this.logger, entityKey).ConfigureAwait(false);
string expectedLogText = $"CircuitBreaker AddFailure triggered";
Assert.Contains(expectedLogText, this.logger.Logs[0], StringComparison.OrdinalIgnoreCase);
Assert.Equal(entityKey, actualEntityId.EntityKey);
req.Dispose();
}
public async Task AddFailure(FailureRequest req)
{
if (State == CircuitState.Open)
{
_log.LogInformation($"Tried to add additional failure to {Entity.Current.EntityKey} that is already open.");
return;
}
FailureWindow.Add(req.RequestId, req);
var cutoff = req.FailureTime.Subtract(windowSize);
// Filter the window only to exceptions within the cutoff timespan
FailureWindow = FailureWindow.Where(p => p.Value.FailureTime >= cutoff).ToDictionary(p => p.Key, p => p.Value);
if (FailureWindow.Count >= failureThreshold)
{
_log.LogCritical($"Break this circuit for entity {Entity.Current.EntityKey}!");
await _durableClient.StartNewAsync(nameof(OpenCircuitOrchestrator.OpenCircuit), req.InstanceId);
// Mark the circuit as "open" (circuit is broken)
State = CircuitState.Open;
}
else
{
_log.LogInformation($"The circuit {Entity.Current.EntityKey} currently has {FailureWindow.Count} exceptions in the window of {windowSize.ToString()}");
}
}
public void Failure(string workerId, string externalTaskId, string errorMessage, int retries, long retryTimeout)
{
var request = new FailureRequest();
request.WorkerId = workerId;
request.ErrorMessage = errorMessage;
request.Retries = retries;
request.RetryTimeout = retryTimeout;
var failureHttpRequest = new HttpClientRequest <FailureRequest>("external-task/" + externalTaskId + "/failure", request);
_httpClientService.PostAsync <FailureRequest>(failureHttpRequest);
}
public void AddFailure(FailureRequest req)
{
var state = disabledProviders.FirstOrDefault(e => e.Name == req.ProviderName);
if (state != null)
{
_logger?.LogInformation($"Tried to add additional failure to {Entity.Current.EntityKey} that is already disabled.");
return;
}
// Counter because the key will be already exist in the dictionary
if (FailureWindow.TryGetValue(req.ProviderName, out var list))
{
list.Add(req);
}
else
{
FailureWindow.Add(req.ProviderName, new List <FailureRequest> {
req
});
}
var timeWindow = TimeSpan.FromSeconds(_configOptions.Value.EmailProvidersSettings.TimeWindowInSeconds);
var threshold = _configOptions.Value.EmailProvidersSettings.Threshold;
var cutoff = req.HappenedAt.Subtract(timeWindow);
// Filter the window only to exceptions within the cutoff timespan
FailureWindow[req.ProviderName].RemoveAll(p => p.HappenedAt < cutoff);
// get Items to delete or postpone
var entityKey = Entity.Current?.EntityKey ?? "";
if (FailureWindow[req.ProviderName].Count >= threshold)
{
_logger?.LogCritical($"{entityKey} Stop using email provider {req.ProviderName} because it exceeded the threshold {threshold}. Number of failure is {FailureWindow[req.ProviderName].Count}");
emailProviders.Remove(req.ProviderName);
var disablePeriod = _configOptions.Value.EmailProvidersSettings.DisablePeriod;
var period = DateTimeOffset.UtcNow.Add(TimeSpan.FromSeconds(disablePeriod));
disabledProviders.Add(new DisabledEmailProviderItem {
Name = req.ProviderName, DueTime = period
});
}
else
{
_logger?.LogInformation($"{entityKey} New failure occurred for provider {req.ProviderName} but it didn't reach the threshold in the timewindow yet.");
}
}
internal void Failure(string workerId, string externalTaskId, string errorMessage, int retries, long retryTimeout)
{
HttpClient http = helper.HttpClient("external-task/" + externalTaskId + "/failure");
FailureRequest request = new FailureRequest();
request.workerId = workerId;
request.errorMessage = errorMessage;
request.retries = retries;
request.retryTimeout = retryTimeout;
HttpResponseMessage response = http.PostAsJsonAsync("", request).Result;
http.Dispose();
if (!response.IsSuccessStatusCode)
{
throw new EngineException("Could not report failure for external Task: " + response.ReasonPhrase);
}
}
/// <summary>
/// Task to Add Failure.
/// </summary>
/// <param name="req">FailureRequest.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
public async Task AddFailure(FailureRequest req)
{
if (req == null)
{
throw new ArgumentNullException(nameof(req));
}
if (this.State == CircuitState.Open)
{
this.log.LogWarning("Tried to add additional failure to {entityKey} that is already open.", entityKey);
return;
}
this.FailureWindow.Add(req.RequestId, req);
var cutoff = req.FailureTime.Subtract(WindowSize);
// Filter the window only to exceptions within the cutoff timespan
this.FailureWindow = this.FailureWindow.Where(p => p.Value.FailureTime >= cutoff).ToDictionary(p => p.Key, p => p.Value);
if (this.FailureWindow.Count >= FailureThreshold)
{
this.log.LogCritical("Break this circuit for entity {entityKey}!", entityKey);
#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task
string instanceId = await this.durableClient.StartNewAsync(nameof(OpenCircuitOrchestrator.OpenCircuit), req);
#pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task
this.log.LogInformation("Started {orchestration} with instance ID {instanceId}.", nameof(OpenCircuitOrchestrator.OpenCircuit), instanceId);
// Mark the circuit as "open" (circuit is broken)
this.State = CircuitState.Open;
}
else
{
this.log.LogInformation(
"The circuit {entityKey} currently has {Count} exceptions in the window of {WindowSize}.",
entityKey,
this.FailureWindow.Count,
WindowSize);
}
}
public void Failure(string workerId, string externalTaskId, string errorMessage, int retries, long retryTimeout)
{
var http = helper.HttpClient("external-task/" + externalTaskId + "/failure");
var request = new FailureRequest();
request.WorkerId = workerId;
request.ErrorMessage = errorMessage;
request.Retries = retries;
request.RetryTimeout = retryTimeout;
var requestContent = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, CamundaClientHelper.CONTENT_TYPE_JSON);
var response = http.PostAsync("", requestContent).Result;
http.Dispose();
if (!response.IsSuccessStatusCode)
{
throw new EngineException("Could not report failure for external Task: " + response.ReasonPhrase);
}
}
public async Task AddFailureShouldCountIncomingEvents1()
{
this.durableClientMock.Setup(service => service.StartNewAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns(Task.FromResult(string.Empty));
CircuitBreakerActor function = new CircuitBreakerActor(this.durableClientMock.Object, this.logger);
FailureRequest req = new FailureRequest
{
RequestId = "RequestId",
FailureTime = DateTime.UtcNow,
InstanceId = "InstanceId",
ResourceId = "ResourceID",
};
req.RequestId = "Req1";
await function.AddFailure(req).ConfigureAwait(false);
string expectedLogText = "currently has 1 exceptions in the window";
Assert.Contains(expectedLogText, this.logger.Logs[0], StringComparison.OrdinalIgnoreCase);
}
public static async Task <HttpResponseMessage> AddFailureFunction(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "CircuitBreaker/{entityKey}/AddFailure")] HttpRequestMessage req,
[DurableClient] IDurableClient client,
ILogger log,
string entityKey)
{
if (req == null)
{
throw new ArgumentNullException(nameof(req));
}
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
string requestBody = await req.Content.ReadAsStringAsync().ConfigureAwait(false);
if (string.IsNullOrEmpty(requestBody))
{
log.LogError("Received an empty request body.");
return(req.CreateErrorResponse(HttpStatusCode.BadRequest, "Request body cannot be empty."));
}
else
{
FailureRequest failureReq = JsonConvert.DeserializeObject <FailureRequest>(requestBody);
log.LogInformation(
"CircuitBreaker AddFailure triggered for {RequestId} with failure time of '{FailureTime}' on instance '{InstanceId}' with specified resource '{ResourceId}'.",
failureReq.RequestId,
failureReq.FailureTime,
failureReq.InstanceId,
failureReq.ResourceId);
var entityId = new EntityId("CircuitBreakerActor", entityKey);
await client.SignalEntityAsync <ICircuitBreakerActor>(entityId, proxy => proxy.AddFailure(failureReq)).ConfigureAwait(false);
return(req.CreateResponse(HttpStatusCode.Accepted));
}
}
public async Task VerifyOpenCircuitFunctionInvokesCallHttpAsync()
{
FailureRequest mockFailureRequest = new FailureRequest
{
FailureTime = DateTime.UtcNow,
InstanceId = "TEST-INSTANCEID",
RequestId = "TEST-REQUESTID",
ResourceId = "TEST-RESOURCEID",
};
this.durableOrchestrationContextMock.Setup(client => client.GetInput <FailureRequest>()).Returns(mockFailureRequest);
this.durableOrchestrationContextMock.Setup(client => client.CallHttpAsync(It.IsAny <DurableHttpRequest>())).Returns(Task.FromResult(new DurableHttpResponse(HttpStatusCode.OK)));
OpenCircuitOrchestrator function = new OpenCircuitOrchestrator();
await function.OpenCircuit(this.durableOrchestrationContextMock.Object, this.logger).ConfigureAwait(false);
this.durableOrchestrationContextMock.Verify(service => service.CallHttpAsync(It.IsAny <DurableHttpRequest>()), Times.Once);
string expectedLogText = "Successfully STOPPED Azure Function with Resource ID";
Assert.Contains(expectedLogText, this.logger.Logs[1], StringComparison.OrdinalIgnoreCase);
}
public async Task ShouldGenerateWarningLogIfCircuitIsAlreadyOpen()
{
this.durableClientMock.Setup(service => service.StartNewAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns(Task.FromResult(string.Empty));
CircuitBreakerActor function = new CircuitBreakerActor(this.durableClientMock.Object, this.logger);
FailureRequest req = new FailureRequest
{
RequestId = "RequestId",
FailureTime = DateTime.UtcNow,
InstanceId = "InstanceId",
ResourceId = "ResourceID",
};
for (int i = 1; i <= 3; i++)
{
req.RequestId = $"Req{i}";
req.FailureTime = req.FailureTime.AddSeconds(10);
await function.AddFailure(req).ConfigureAwait(false);
}
string expectedLogText = "Tried to add additional failure to";
Assert.Contains(expectedLogText, this.logger.Logs[3], StringComparison.OrdinalIgnoreCase);
}
public async Task Run(
[ServiceBusTrigger("%ServiceBusTopic%", "%ServiceBusSubscription%", Connection = "ServiceBusConnection")] Message message,
ILogger log)
{
IconicsFault iconicsFault = null;
string messageContents = null;
if (message == null)
{
// This should never happen.
throw new ArgumentNullException(nameof(message));
}
try
{
if (message != null)
{
// deserialize message contents into iconics fault class
messageContents = Encoding.UTF8.GetString(message.Body);
iconicsFault = JsonConvert.DeserializeObject <IconicsFault>(messageContents);
log.LogInformation(
"Received ICONICS fault with FaultName '{faultName}' and FaultActiveTime of '{faultActiveTime}' and AssetPath of '{assetPath}'.",
iconicsFault.FaultName,
iconicsFault.FaultActiveTime,
iconicsFault.AssetPath);
}
}
catch (JsonException e)
{
// message could not be deserialized, send to error queue
log.LogError(
"Exception while processing message: {message}, exception :{exceptionMessage}. Sending message to the error queue.",
message,
e.Message);
await this.errorQueueService.SendMessageToErrorQueueAsync(messageContents, e.Message).ConfigureAwait(false);
return;
}
// validate fault data properties
if (!this.validationService.IsValidFault(iconicsFault, out string validationMessage))
{
// send message to error queue
log.LogError(Resources.FaultValidationFailure);
await this.errorQueueService.SendMessageToErrorQueueAsync(messageContents, validationMessage).ConfigureAwait(false);
}
else
{
if (this.dynamicsEntityService.IsCdsServiceReady())
{
// If Iconic Fault State is "Active", Function should create new IoT Alert in Dynamics
if (string.IsNullOrEmpty(iconicsFault.FaultState) || iconicsFault.FaultState.Equals("Active", StringComparison.OrdinalIgnoreCase))
{
// Create Dynamics asset (if necessary) and alert
Guid assetId = this.dynamicsEntityService.CreateAssetIfNotExists(iconicsFault, log);
Guid alertId = this.dynamicsEntityService.CreateAlert(iconicsFault, assetId, log);
// Determine the elapsed time to create the alert.
// Lack of an EventEnqueuedUtcTime should not prevent the alert from being created. The field impacts telemetry only.
var parseSucceeded = DateTime.TryParse(iconicsFault.EventEnqueuedUtcTime, out DateTime enqueuedDateTimeUtc);
if (!parseSucceeded)
{
log.LogWarning("Unable to determine IoT alert created elapsed time due to missing fault event enqueued time.");
}
else
{
var et = new EventTelemetry("IoTAlertCreated");
et.Metrics.Add("AlertCreatedElapsedTimeMs", (DateTime.UtcNow - enqueuedDateTimeUtc).TotalMilliseconds);
this.telemetryClient.TrackEvent(et);
}
}
else if (iconicsFault.FaultState.Equals("InActive", StringComparison.OrdinalIgnoreCase))
{
// If if Iconic Fault State is "InActive", Function should find existing IoT Alert and update Alert Data and Status of an IoT Alert to InActive in Dynamics
// Update IoT Alert State to InActive and Alert Data with latest fault data
this.dynamicsEntityService.UpdateIoTAlert(iconicsFault, log, (int)IotAlertStateCode.Inactive, (int)IotAlertStatusCode.Inactive);
}
}
else
{
log.LogError("{msg}, error: {error}", Resources.CDSConnectionFailure, this.dynamicsEntityService.RetrieveCdsError());
FailureRequest failureReq = new FailureRequest
{
// RequestId = $"{message?.MessageId}:{Guid.NewGuid()}",
RequestId = Guid.NewGuid().ToString(),
FailureTime = DateTime.UtcNow,
InstanceId = this.instanceId.Id,
ResourceId = ResourceId,
};
string appName = ResourceId.Split('/').Last();
string circuitBreakerBaseUri = $"{CircuitRequestUri}/{appName}/AddFailure";
var response = await this.httpClient.PostAsJsonAsync($"{circuitBreakerBaseUri}?code={CircuitCode}", failureReq).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
log.LogError("Failed to invoke Circuit Breaker function, {circuitBreakerBaseUri}. Status code: {code}. Reason: '{reason}'", circuitBreakerBaseUri, response.StatusCode, response.ReasonPhrase);
}
else
{
log.LogInformation("Successfully invoked Circuit Breaker function, {circuitBreakerBaseUri}", circuitBreakerBaseUri);
}
// Throw an exception here in order to instruct the binding to Abandon the message (for the function to fail).
// See https://docs.microsoft.com/en-us/azure/azure-functions/functions-bindings-service-bus-trigger?tabs=csharp#peeklock-behavior
throw new ApplicationException($"{Resources.CDSConnectionFailure}, error: {this.dynamicsEntityService.RetrieveCdsError()}");
}
}
}