/// <summary>
/// Attempts to acquire a lease based on the provided lease policy.
/// </summary>
/// <param name="leasePolicy">The configuration details for the lease.</param>
/// <param name="leaseId">The id of the currently acquired lease.</param>
/// <returns>A task for the async operation.</returns>
public async Task <string> AcquireAsync(ILeasePolicy leasePolicy, string leaseId)
{
this.LeasePolicy = leasePolicy;
await this.InitialiseAsync();
if (!this.blob.Exists())
{
await Retriable.RetryAsync(async() =>
{
using (var ms = new MemoryStream())
{
await this.blob.UploadFromStreamAsync(ms);
}
});
}
try
{
return(await Retriable.RetryAsync(
() => this.blob.AcquireLeaseAsync(leasePolicy.Duration, leaseId),
new CancellationToken(),
new Count(10),
new DoNotRetryOnConflictPolicy()));
}
catch (StorageException exception)
{
if (exception.RequestInformation.HttpStatusCode == 409)
{
throw new LeaseAcquisitionUnsuccessfulException(this.LeasePolicy, exception);
}
throw;
}
}
private async Task <DataflowContext> DownloadFeedAsync(DataflowContext context)
{
try
{
await Retriable.RetryAsync(
async() =>
{
if (File.Exists(context.Destination))
{
context.AlreadyDownloaded = true;
Console.WriteLine("Already Downloaded: " + context.Destination);
return;
}
else
{
var fileInfo = new FileInfo(context.Destination);
if (!fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
}
using (HttpClient client = this.httpClientFactory.CreateClient())
{
var request = new HttpRequestMessage(HttpMethod.Get, context.Source);
HttpResponseMessage response = await client.SendAsync(request).ConfigureAwait(false);
using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
{
using (Stream streamToWriteTo = File.Open(context.Destination, FileMode.Create))
{
await streamToReadFrom.CopyToAsync(streamToWriteTo).ConfigureAwait(false);
}
}
Console.WriteLine("Downloaded: " + context.Destination);
response.EnsureSuccessStatusCode();
}
},
CancellationToken.None,
new Backoff(5, TimeSpan.FromSeconds(1)),
new AnyException()).ConfigureAwait(false);
context.IsFaulted = false;
}
catch (Exception ex)
{
context.IsFaulted = true;
context.FaultError = ex.Message;
Console.WriteLine("Error Downloading: " + context.Destination);
}
return(context);
}
public Task ThenWithinSecondsTheFirstNotificationsStoredInTheTransientTenantForTheUserWithIdHaveTheDeliveryStatusForTheDeliveryChannelWithId(
int timeoutSeconds,
int count,
string userId,
UserNotificationDeliveryStatus expectedDeliveryStatus,
string deliveryChannelId)
{
var tokenSource = new CancellationTokenSource();
tokenSource.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds));
return(Retriable.RetryAsync(
async() =>
{
GetNotificationsResult userNotifications = await this.GetNotificationsForUserAsync(userId, count).ConfigureAwait(false);
foreach (UserNotification current in userNotifications.Results)
{
if (current.GetDeliveryStatusForChannel(deliveryChannelId) != expectedDeliveryStatus)
{
throw new Exception($"Notification with Id '{current.Id}' has delivery status '{current.GetDeliveryStatusForChannel(deliveryChannelId)}' instead of expected value '{expectedDeliveryStatus}'");
}
}
},
tokenSource.Token,
new Linear(TimeSpan.FromSeconds(5), int.MaxValue),
new AnyExceptionPolicy(),
false));
}
private async Task InitialiseAsync()
{
if (!this.initialised)
{
try
{
this.storageAccount = this.GetStorageAccount();
this.client = Retriable.Retry(() => this.storageAccount.CreateCloudBlobClient());
string containerName = this.GetContainerName();
this.container = Retriable.Retry(() => this.client.GetContainerReference(containerName));
if (await this.container.CreateIfNotExistsAsync().ConfigureAwait(false))
{
var containerPermissions = new BlobContainerPermissions {
PublicAccess = BlobContainerPublicAccessType.Off
};
await Retriable.RetryAsync(() => this.container.SetPermissionsAsync(containerPermissions)).ConfigureAwait(false);
}
this.initialised = true;
}
catch (Exception ex)
{
throw new InitializationFailureException("Initialization failed.", ex);
}
}
}
private static async Task RunAsync()
{
ISomeService someTasks = new MyService();
var result = await Retriable.RetryAsync(() => SomeFuncAsync(someTasks));
Console.WriteLine(result);
}
/// <summary>
/// Attempts to extend the lease based on the lease policy provided to initially acquire it.
/// </summary>
/// <param name="leaseId">The id of the lease to attempt to extend.</param>
/// <remarks>A valid lease and lease policy must exist for this operation to execute. An InvalidOperationException will be thrown otherwise.</remarks>
/// <returns>A task for the async operation.</returns>
public async Task ExtendAsync(string leaseId)
{
await this.InitialiseAsync();
await Retriable.RetryAsync(() => this.blob.RenewLeaseAsync(new AccessCondition {
LeaseId = leaseId
}));
}
/// <summary>
/// Executes the supplied method with standard retry strategy and policy.
/// </summary>
/// <typeparam name="T">The type of the return value.</typeparam>
/// <param name="asyncMethod">The method to execute.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
public static Task <T> ExecuteWithStandardTestRetryRulesAsync <T>(Func <Task <T> > asyncMethod)
{
return(Retriable.RetryAsync(
asyncMethod,
CancellationToken.None,
new UseCosmosRetryAfterHeaderStrategy(TimeSpan.FromSeconds(2), 5),
RetryOnCosmosRequestRateExceededPolicy.Instance,
false));
}
/// <summary>
/// Executes the supplied method with standard retry strategy and policy.
/// </summary>
/// <param name="asyncMethod">The method to execute.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
public static Task ExecuteWithStandardTestRetryRulesAsync(Func <Task> asyncMethod)
{
return(Retriable.RetryAsync(
asyncMethod,
CancellationToken.None,
new Linear(TimeSpan.FromSeconds(5), 5),
RetryOnCosmosRequestRateExceededPolicy.Instance,
false));
}
/// <inheritdoc/>
public async Task UpsertWorkflowAsync(Workflow workflow, string partitionKey = null)
{
await Retriable.RetryAsync(
() => this.UpsertWorkflowCoreAsync(workflow),
CancellationToken.None,
new Backoff(5, TimeSpan.FromSeconds(1)),
DoNotRetryWhenFutile.Instance)
.ConfigureAwait(false);
}
/// <inheritdoc/>
public async Task <Workflow> GetWorkflowAsync(string workflowId, string partitionKey = null)
{
return(await Retriable.RetryAsync(
() => this.GetWorkflowCoreAsync(workflowId),
CancellationToken.None,
new Backoff(5, TimeSpan.FromSeconds(1)),
DoNotRetryWhenFutile.Instance)
.ConfigureAwait(false));
}
/// <summary>
/// Enumerate the entities matching a particular query.
/// </summary>
/// <typeparam name="T">The type of entity to enumerate.</typeparam>
/// <param name="container">The Cosmos container against which to execute the query.</param>
/// <param name="queryDefinition">The query definition.</param>
/// <param name="actionAsync">The action to execute.</param>
/// <param name="requestOptions">Request options for the query.</param>
/// <param name="maxBatchCount">The maximum number of batches to process.</param>
/// <param name="continuationToken">The continuation token from which to resume processing.</param>
/// <param name="cancellationToken">A cancellation token to terminate the option early.</param>
/// <returns>A <see cref="Task"/> which provides a continuation token if it terminates before .</returns>
public static async Task <string?> ForEachAsync <T>(this Container container, QueryDefinition queryDefinition, Func <T, Task> actionAsync, QueryRequestOptions?requestOptions = null, int?maxBatchCount = null, string?continuationToken = null, CancellationToken cancellationToken = default)
{
if (container is null)
{
throw new ArgumentNullException(nameof(container));
}
if (queryDefinition == null)
{
throw new ArgumentNullException(nameof(queryDefinition));
}
if (actionAsync == null)
{
throw new ArgumentNullException(nameof(actionAsync));
}
FeedIterator <T> iterator = container.GetItemQueryIterator <T>(queryDefinition, continuationToken, requestOptions);
int batchCount = 0;
string?previousContinuationToken = null;
string?responseContinuationToken = null;
try
{
while (iterator.HasMoreResults && (!maxBatchCount.HasValue || batchCount < maxBatchCount.Value))
{
batchCount++;
FeedResponse <T> response = await Retriable.RetryAsync(
() => iterator.ReadNextAsync(cancellationToken),
CancellationToken.None,
new Backoff(3, TimeSpan.FromSeconds(1)),
RetryOnBusyPolicy.Instance,
false)
.ConfigureAwait(false);
previousContinuationToken = responseContinuationToken;
responseContinuationToken = response.ContinuationToken;
foreach (T item in response)
{
await actionAsync(item).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
}
}
catch (OperationCanceledException)
{
// We cancelled the operation so we revert to the previous continuation token to allow reprocessing from the previous batch
responseContinuationToken = previousContinuationToken;
}
return(responseContinuationToken);
}
/// <inheritdoc/>
public Task UpsertWorkflowAsync(Workflow workflow, string partitionKey = null)
{
return(Retriable.RetryAsync(() =>
{
if (string.IsNullOrEmpty(workflow.ETag))
{
return this.CreateWorkflowAsync(workflow, partitionKey);
}
return this.UpdateWorkflowAsync(workflow, partitionKey);
}));
}
public async Task ThenAfterSecondsAtMostTheWorkflowInstanceWithIdShouldBeInTheStateWithName(string instanceId, string expectedStateName, int maxWaitTime)
{
var tokenSource = new CancellationTokenSource();
tokenSource.CancelAfter(TimeSpan.FromSeconds(maxWaitTime));
await Retriable.RetryAsync(
() => this.VerifyWorkflowInstanceState(instanceId, expectedStateName, false),
tokenSource.Token,
new Linear(TimeSpan.FromSeconds(1), int.MaxValue),
new AnyExceptionPolicy(),
false).ConfigureAwait(false);
}
public async Task ThenAfterSecondsAtMostThereShouldBeAWorkflowInstanceWithTheIdInTheWorkflowInstanceStore(string instanceId, int maximumWaitTime)
{
var tokenSource = new CancellationTokenSource();
tokenSource.CancelAfter(TimeSpan.FromSeconds(maximumWaitTime));
await Retriable.RetryAsync(
() => this.GetWorkflowInstance(instanceId, false),
tokenSource.Token,
new Linear(TimeSpan.FromSeconds(1), int.MaxValue),
new AnyExceptionPolicy(),
false).ConfigureAwait(false);
}
/// <inheritdoc/>
public async Task <int> GetMatchingWorkflowInstanceCountForSubjectsAsync(IEnumerable <string> subjects)
{
QueryDefinition spec = BuildFindInstanceIdsSpec(subjects, 1, 0, true);
FeedIterator <int> iterator = this.Container.GetItemQueryIterator <int>(spec, null, new QueryRequestOptions {
MaxItemCount = 1
});
// There will always be a result so we don't need to check...
FeedResponse <int> result = await Retriable.RetryAsync(() => iterator.ReadNextAsync()).ConfigureAwait(false);
return(result.First());
}
private static async Task RunInlineAsync()
{
ISomeService someTasks = new MyService();
var result = await Retriable.RetryAsync(async delegate
{
var response = await someTasks.FirstTaskAsync();
return(await someTasks.SecondTaskAsync(response));
});
Console.WriteLine(result);
}
/// <inheritdoc/>
public async Task DeleteWorkflowInstanceAsync(string workflowInstanceId, string partitionKey = null)
{
try
{
await Retriable.RetryAsync(() =>
this.Container.DeleteItemAsync <WorkflowInstance>(
workflowInstanceId,
new PartitionKey(partitionKey ?? workflowInstanceId)))
.ConfigureAwait(false);
}
catch (CosmosException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new WorkflowInstanceNotFoundException($"The workflow instance with id {workflowInstanceId} was not found", ex);
}
}
private async Task InitialiseAsync()
{
if (!this.isInitialised)
{
var storageAccount = CloudStorageAccount.Parse(Configuration.GetSettingFor(this.connectionStringProvider.ConnectionStringKey));
var client = storageAccount.CreateCloudBlobClient();
var container = client.GetContainerReference("endjin-leasing-leases");
await Retriable.RetryAsync(container.CreateIfNotExistsAsync);
this.blob = container.GetBlockBlobReference(this.LeasePolicy.Name.ToLowerInvariant());
this.isInitialised = true;
}
}
/// <inheritdoc/>
public async Task <Workflow> GetWorkflowAsync(string workflowId, string partitionKey = null)
{
try
{
ItemResponse <Workflow> itemResponse = await Retriable.RetryAsync(() =>
this.Container.ReadItemAsync <Workflow>(
workflowId,
new PartitionKey(partitionKey ?? workflowId)))
.ConfigureAwait(false);
return(itemResponse.Resource);
}
catch (CosmosException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new WorkflowNotFoundException($"The workflow with id {workflowId} was not found", ex);
}
}
public async Task <Lease> GetAutoRenewingLeaseWithOptionsAsync(CancellationToken cancellationToken, string leaseName, string actorName = "")
{
var leaseProvider = this.leaseProviderFactory.Create();
this.CheckProperties();
var lease = new Lease(leaseProvider, this.leasePolicyValidator);
await Retriable.RetryAsync(async() =>
{
await this.AcquireLeaseAndStartRenewingAsync(cancellationToken, leaseProvider, lease);
},
cancellationToken,
this.RetryStrategy,
this.RetryPolicy);
return(lease);
}
/// <inheritdoc/>
public async Task UpsertWorkflowInstanceAsync(WorkflowInstance workflowInstance, string partitionKey = null)
{
try
{
await Retriable.RetryAsync(() =>
this.Container.UpsertItemAsync(
workflowInstance,
new PartitionKey(partitionKey ?? workflowInstance.Id),
new ItemRequestOptions {
IfMatchEtag = workflowInstance.ETag
}))
.ConfigureAwait(false);
}
catch (CosmosException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new WorkflowInstanceConflictException($"The workflow instance with id {workflowInstance.Id} was already modified.", ex);
}
}
/// <inheritdoc/>
public async Task ExtendAsync(Lease lease)
{
if (lease is null)
{
throw new ArgumentNullException(nameof(lease));
}
this.logger.LogDebug($"Extending lease for '{lease.LeasePolicy.ActorName}' with name '{lease.LeasePolicy.Name}', duration '{lease.LeasePolicy.Duration}', and actual id '{lease.Id}'");
await this.InitialiseAsync().ConfigureAwait(false);
CloudBlockBlob blob = this.container !.GetBlockBlobReference(lease.LeasePolicy.Name.ToLowerInvariant());
await Retriable.RetryAsync(() => blob.RenewLeaseAsync(new AccessCondition {
LeaseId = lease.Id
})).ConfigureAwait(false);
(lease as AzureLease)?.SetLastAcquired(DateTimeOffset.Now);
this.logger.LogDebug($"Extended lease for '{lease.LeasePolicy.ActorName}' with name '{lease.LeasePolicy.Name}', duration '{lease.LeasePolicy.Duration}', and actual id '{lease.Id}'");
}
private async Task <Tuple <bool, T> > TryAcquireLeaseAndExecuteAsync <T>(Func <CancellationToken, Task <T> > action, CancellationTokenSource cancellationTokenSource, ILeaseProvider leaseProvider)
{
try
{
this.CheckProperties();
var result = await Retriable.RetryAsync(
() => this.AcquireLeaseAndExecuteInnerAsync(action, cancellationTokenSource, leaseProvider),
cancellationTokenSource.Token,
this.RetryStrategy,
this.RetryPolicy);
return(new Tuple <bool, T>(true, result));
}
catch (LeaseAcquisitionUnsuccessfulException)
{
return(new Tuple <bool, T>(false, default(T)));
}
}
private async Task PublishWithRetryAsync <T>(string source, string subject, T cloudEvent, WorkflowEventSubscription destination)
{
try
{
await Retriable.RetryAsync(() => this.PublishAsync(source, subject, cloudEvent, destination)).ConfigureAwait(false);
}
catch (Exception ex)
{
// In this "v1" solution to event publishing, we don't want exceptions to propagate outside the
// publisher because we don't want failure in event publishing to break anything.
this.logger.LogError(
ex,
"Unexpected exception when trying to a CloudEvent with source '{source}', '{subject}' and destinationUrl '{destinationUrl}' with authentication resource '{msiAuthenticationResource}'.",
source,
subject,
destination?.ExternalUrl,
destination?.MsiAuthenticationResource);
}
}
/// <inheritdoc/>
public async Task <IEnumerable <string> > GetMatchingWorkflowInstanceIdsForSubjectsAsync(
IEnumerable <string> subjects,
int pageSize,
int pageNumber)
{
QueryDefinition spec = BuildFindInstanceIdsSpec(subjects, pageSize, pageNumber);
FeedIterator <dynamic> iterator = this.Container.GetItemQueryIterator <dynamic>(spec);
var matchingIds = new List <string>();
while (iterator.HasMoreResults)
{
FeedResponse <dynamic> results = await Retriable.RetryAsync(() => iterator.ReadNextAsync()).ConfigureAwait(false);
matchingIds.AddRange(results.Select(x => (string)x.id));
}
return(matchingIds);
}
private Task LongRunningOperationPropertyCheck(Uri location, string operationPropertyPath, int timeoutSeconds, Action <string> testValue)
{
var tokenSource = new CancellationTokenSource();
tokenSource.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds));
return(Retriable.RetryAsync(
async() =>
{
HttpResponseMessage response = await HttpClient.GetAsync(location).ConfigureAwait(false);
string responseBody = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
var operation = JObject.Parse(responseBody);
JToken targetToken = operation.SelectToken(operationPropertyPath);
string currentValue = targetToken.Value <string>();
testValue(currentValue);
},
tokenSource.Token,
new Linear(TimeSpan.FromSeconds(5), int.MaxValue),
new AnyExceptionPolicy(),
false));
}
private async Task <bool> TryAcquireLeaseAndExecuteAsync <T>(Func <CancellationToken, T, Task> action, CancellationTokenSource cancellationTokenSource, ILeaseProvider leaseProvider, T arg)
{
try
{
this.CheckProperties();
await Retriable.RetryAsync(async() =>
{
await this.AcquireLeaseAndExecuteInnerAsync(action, cancellationTokenSource, leaseProvider, arg);
},
cancellationTokenSource.Token,
this.RetryStrategy,
this.RetryPolicy);
return(true);
}
catch (LeaseAcquisitionUnsuccessfulException)
{
return(false);
}
}
/// <inheritdoc/>
public async Task <WorkflowInstance> StartWorkflowInstanceAsync(
string workflowId,
string workflowPartitionKey = null,
string instanceId = null,
string instancePartitionKey = null,
IDictionary <string, string> context = null)
{
// We need to clone the original context here in case it's modified by any of the code which creates the
// workflow instance.
var originalSuppliedContext = context?.ToDictionary(x => x.Key, x => x.Value);
WorkflowInstance newInstance = await Retriable.RetryAsync(() =>
this.CreateWorkflowInstanceAsync(workflowId, workflowPartitionKey, instanceId, instancePartitionKey, context)).ConfigureAwait(false);
// We publish the CloudEvent outside the Retry block because:
// a) we only want to publish the event once the instance is created, and
// b) we don't want a failure in CloudEvent publishing to cause a retry
// of the whole process. The ICloudEventPublisher implementation is expected
// to provide its own retry mechanism.
Workflow workflow = await this.workflowStore.GetWorkflowAsync(newInstance.WorkflowId).ConfigureAwait(false);
var workflowEventData = new WorkflowInstanceCreationCloudEventData(newInstance.Id)
{
NewContext = newInstance.Context,
NewState = newInstance.StateId,
NewStatus = newInstance.Status,
SuppliedContext = originalSuppliedContext,
WorkflowId = newInstance.WorkflowId,
};
await this.cloudEventPublisher.PublishWorkflowEventDataAsync(
this.cloudEventSource,
WorkflowEventTypes.InstanceCreated,
newInstance.Id,
workflowEventData.ContentType,
workflowEventData,
workflow.WorkflowEventSubscriptions).ConfigureAwait(false);
return(newInstance);
}
/// <inheritdoc/>
public async Task ReleaseAsync(Lease lease)
{
if (lease is null)
{
throw new ArgumentNullException(nameof(lease));
}
if (lease is not AzureLease al)
{
throw new ArgumentException("Only Leases of type 'AzureLease' can be released by the AzureLeaseProvider.");
}
this.logger.LogDebug($"Releasing lease for '{lease.LeasePolicy.ActorName}' with name '{lease.LeasePolicy.Name}', duration '{lease.LeasePolicy.Duration}', and actual id '{lease.Id}'");
await this.InitialiseAsync().ConfigureAwait(false);
CloudBlockBlob blob = this.container !.GetBlockBlobReference(lease.LeasePolicy.Name.ToLowerInvariant());
await Retriable.RetryAsync(() => blob.ReleaseLeaseAsync(new AccessCondition {
LeaseId = lease.Id
})).ConfigureAwait(false);
al.SetLastAcquired(null);
this.logger.LogDebug($"Released lease for '{lease.LeasePolicy.ActorName}' with name '{lease.LeasePolicy.Name}', duration '{lease.LeasePolicy.Duration}', and actual id '{lease.Id}'");
}
public async Task <Lease> GetAutoRenewingLeaseAsync(CancellationToken cancellationToken, string leaseName, string actorName = "")
{
var leaseProvider = this.leaseProviderFactory.Create();
this.LeasePolicy = new LeasePolicy {
Duration = leaseProvider.DefaultLeaseDuration, Name = leaseName, ActorName = actorName
};
this.RetryStrategy = new Linear(TimeSpan.FromSeconds(Math.Round(leaseProvider.DefaultLeaseDuration.TotalSeconds / 3)), int.MaxValue);
this.RetryPolicy = new RetryUntilLeaseAcquiredPolicy();
this.CheckProperties();
var lease = new Lease(leaseProvider, this.leasePolicyValidator);
await Retriable.RetryAsync(async() =>
{
await this.AcquireLeaseAndStartRenewingAsync(cancellationToken, leaseProvider, lease);
},
cancellationToken,
this.RetryStrategy,
this.RetryPolicy);
return(lease);
}
