public BlobLeaseManager(
AzureStorageOrchestrationServiceSettings settings,
string leaseContainerName,
string blobPrefix,
string leaseType,
CloudBlobClient storageClient,
bool skipBlobContainerCreation,
AzureStorageOrchestrationServiceStats stats)
{
this.settings = settings;
this.storageAccountName = storageClient.Credentials.AccountName;
this.taskHubName = settings.TaskHubName;
this.workerName = settings.WorkerId;
this.leaseContainerName = leaseContainerName;
this.blobPrefix = blobPrefix;
this.leaseType = leaseType;
this.storageClient = storageClient;
this.leaseInterval = settings.LeaseInterval;
this.skipBlobContainerCreation = skipBlobContainerCreation;
this.renewRequestOptions = new BlobRequestOptions {
ServerTimeout = settings.LeaseRenewInterval
};
this.stats = stats ?? new AzureStorageOrchestrationServiceStats();
this.Initialize();
}
public BlobLeaseManager(
string taskHubName,
string workerName,
string leaseContainerName,
string blobPrefix,
string consumerGroupName,
CloudBlobClient storageClient,
TimeSpan leaseInterval,
TimeSpan renewInterval,
bool skipBlobContainerCreation,
AzureStorageOrchestrationServiceStats stats)
{
this.storageAccountName = storageClient.Credentials.AccountName;
this.taskHubName = taskHubName;
this.workerName = workerName;
this.leaseContainerName = leaseContainerName;
this.blobPrefix = blobPrefix;
this.consumerGroupName = consumerGroupName;
this.storageClient = storageClient;
this.leaseInterval = leaseInterval;
this.renewInterval = renewInterval;
this.skipBlobContainerCreation = skipBlobContainerCreation;
this.renewRequestOptions = new BlobRequestOptions {
ServerTimeout = renewInterval
};
this.stats = stats ?? new AzureStorageOrchestrationServiceStats();
this.Initialize();
}
public LegacyPartitionManager(
AzureStorageOrchestrationService service,
AzureStorageOrchestrationServiceSettings settings,
CloudStorageAccount account,
AzureStorageOrchestrationServiceStats stats)
{
this.service = service;
this.settings = settings;
this.stats = stats;
this.leaseManager = new BlobLeaseManager(
settings,
settings.TaskHubName.ToLowerInvariant() + "-leases",
string.Empty,
"default",
account.CreateCloudBlobClient(),
skipBlobContainerCreation: false,
stats);
this.leaseCollectionManager = new LeaseCollectionBalancer <BlobLease>(
"default",
settings,
account.Credentials.AccountName,
leaseManager,
new LeaseCollectionBalancerOptions
{
AcquireInterval = settings.LeaseAcquireInterval,
RenewInterval = settings.LeaseRenewInterval,
LeaseInterval = settings.LeaseInterval,
ShouldStealLeases = true,
});
}
public AppLeaseManager(
AzureStorageOrchestrationServiceSettings settings,
string accountName,
CloudBlobClient storageClient,
string appLeaseContainerName,
AppLeaseOptions options,
AzureStorageOrchestrationServiceStats stats)
{
this.settings = settings;
this.accountName = accountName;
this.taskHub = settings.TaskHubName;
this.workerName = settings.WorkerId;
this.storageClient = storageClient;
this.appLeaseContainerName = appLeaseContainerName;
this.appName = settings.AppName;
this.options = options;
this.stats = stats ?? new AzureStorageOrchestrationServiceStats();
this.isLeaseOwner = false;
this.appLeaseContainer = this.storageClient.GetContainerReference(this.appLeaseContainerName);
using (MD5 md5 = MD5.Create())
{
byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(this.appName));
this.appLeaseId = new Guid(hash).ToString();
}
}
internal async Task DeleteLargeMessageBlobs(string instanceId, AzureStorageOrchestrationServiceStats stats)
{
var blobForDeletionTaskList = new List <Task>();
if (!await this.cloudBlobContainer.ExistsAsync())
{
return;
}
CloudBlobDirectory instanceDirectory = this.cloudBlobContainer.GetDirectoryReference(instanceId);
BlobContinuationToken blobContinuationToken = null;
while (true)
{
BlobResultSegment segment = await instanceDirectory.ListBlobsSegmentedAsync(blobContinuationToken);
stats.StorageRequests.Increment();
foreach (IListBlobItem blobListItem in segment.Results)
{
var cloudBlockBlob = blobListItem as CloudBlockBlob;
CloudBlockBlob blob = this.cloudBlobContainer.GetBlockBlobReference(cloudBlockBlob?.Name);
blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync());
}
await Task.WhenAll(blobForDeletionTaskList);
stats.StorageRequests.Increment(blobForDeletionTaskList.Count);
if (blobContinuationToken == null)
{
break;
}
}
}
public AppLeaseManager(
IPartitionManager partitionManager,
AzureStorageOrchestrationServiceSettings settings,
string accountName,
CloudBlobClient storageClient,
string appLeaseContainerName,
string appLeaseInfoBlobName,
AppLeaseOptions options,
AzureStorageOrchestrationServiceStats stats)
{
this.partitionManager = partitionManager;
this.settings = settings;
this.accountName = accountName;
this.storageClient = storageClient;
this.appLeaseContainerName = appLeaseContainerName;
this.appLeaseInfoBlobName = appLeaseInfoBlobName;
this.options = options;
this.stats = stats ?? new AzureStorageOrchestrationServiceStats();
this.taskHub = settings.TaskHubName;
this.workerName = settings.WorkerId;
this.appName = settings.AppName;
this.appLeaseIsEnabled = this.settings.UseAppLease;
this.appLeaseContainer = this.storageClient.GetContainerReference(this.appLeaseContainerName);
this.appLeaseInfoBlob = this.appLeaseContainer.GetBlockBlobReference(this.appLeaseInfoBlobName);
using (MD5 md5 = MD5.Create())
{
byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(this.appName));
this.appLeaseId = new Guid(hash).ToString();
}
this.isLeaseOwner = false;
this.shutdownCompletedEvent = new AsyncManualResetEvent();
}
internal AzureTableTrackingStore(
AzureStorageOrchestrationServiceStats stats,
CloudTable instancesTable
)
{
this.stats = stats;
this.InstancesTable = instancesTable;
}
public WorkItemQueue(
CloudQueue storageQueue,
AzureStorageOrchestrationServiceSettings settings,
AzureStorageOrchestrationServiceStats stats,
MessageManager messageManager)
: base(storageQueue, settings, stats, messageManager)
{
}
public Queue(AzureStorageClient azureStorageClient, CloudQueueClient queueClient, string queueName)
{
this.azureStorageClient = azureStorageClient;
this.queueClient = queueClient;
this.stats = this.azureStorageClient.Stats;
this.Name = queueName;
this.cloudQueue = this.queueClient.GetQueueReference(this.Name);
}
public Table(AzureStorageClient azureStorageClient, CloudTableClient tableClient, string tableName)
{
this.azureStorageClient = azureStorageClient;
this.tableClient = tableClient;
this.Name = tableName;
this.stats = this.azureStorageClient.Stats;
this.cloudTable = this.tableClient.GetTableReference(this.Name);
}
public ControlQueue(
CloudQueue storageQueue,
AzureStorageOrchestrationServiceSettings settings,
AzureStorageOrchestrationServiceStats stats,
MessageManager messageManager)
: base(storageQueue, settings, stats, messageManager)
{
this.releaseTokenSource = new CancellationTokenSource();
this.releaseCancellationToken = this.releaseTokenSource.Token;
}
public ControlQueue(
AzureStorageClient azureStorageClient,
string queueName,
MessageManager messageManager)
: base(azureStorageClient, queueName, messageManager)
{
this.releaseTokenSource = new CancellationTokenSource();
this.releaseCancellationToken = this.releaseTokenSource.Token;
this.stats = this.azureStorageClient.Stats;
}
public BlobLease(
string partitionId,
CloudBlobDirectory leaseDirectory,
string accountName,
AzureStorageOrchestrationServiceStats stats,
AzureStorageOrchestrationServiceSettings settings)
{
this.PartitionId = partitionId;
this.Blob = leaseDirectory.GetBlockBlobReference(partitionId);
this.accountName = accountName;
this.stats = stats;
this.settings = settings;
}
public OrchestrationSessionManager(
string storageAccountName,
AzureStorageOrchestrationServiceSettings settings,
AzureStorageOrchestrationServiceStats stats,
ITrackingStore trackingStore)
{
this.storageAccountName = storageAccountName;
this.settings = settings;
this.stats = stats;
this.trackingStore = trackingStore;
this.fetchRuntimeStateQueue = new DispatchQueue(this.settings.MaxStorageOperationConcurrency);
}
internal async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, AzureStorageOrchestrationServiceStats stats)
{
var blobForDeletionTaskList = new List <Task>();
if (!await this.cloudBlobContainer.ExistsAsync())
{
return;
}
CloudBlobDirectory instanceDirectory = this.cloudBlobContainer.GetDirectoryReference(sanitizedInstanceId);
BlobContinuationToken blobContinuationToken = null;
while (true)
{
BlobResultSegment segment = await TimeoutHandler.ExecuteWithTimeout(
operationName : "DeleteLargeMessageBlobs",
account : cloudBlobContainer?.ServiceClient?.Credentials?.AccountName,
settings : this.settings,
operation : (context, timeoutToken) =>
{
return(instanceDirectory.ListBlobsSegmentedAsync(
useFlatBlobListing: true,
blobListingDetails: BlobListingDetails.Metadata,
maxResults: null,
currentToken: blobContinuationToken,
options: null,
operationContext: context,
cancellationToken: timeoutToken));
});
stats.StorageRequests.Increment();
foreach (IListBlobItem blobListItem in segment.Results)
{
var cloudBlockBlob = blobListItem as CloudBlockBlob;
CloudBlockBlob blob = this.cloudBlobContainer.GetBlockBlobReference(cloudBlockBlob?.Name);
blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync());
}
await Task.WhenAll(blobForDeletionTaskList);
stats.StorageRequests.Increment(blobForDeletionTaskList.Count);
if (blobContinuationToken == null)
{
break;
}
}
}
public TaskHubQueue(
CloudQueue storageQueue,
AzureStorageOrchestrationServiceSettings settings,
AzureStorageOrchestrationServiceStats stats,
MessageManager messageManager)
{
this.storageQueue = storageQueue;
this.storageAccountName = storageQueue.ServiceClient.Credentials.AccountName;
this.settings = settings;
this.stats = stats;
this.messageManager = messageManager;
TimeSpan minPollingDelay = TimeSpan.FromMilliseconds(50);
TimeSpan maxPollingDelay = AzureStorageOrchestrationService.MaxQueuePollingDelay;
this.backoffHelper = new BackoffPollingHelper(minPollingDelay, maxPollingDelay);
}
public AzureTableTrackingStore(
AzureStorageOrchestrationServiceSettings settings,
MessageManager messageManager,
AzureStorageOrchestrationServiceStats stats)
{
this.settings = settings;
this.messageManager = messageManager;
this.stats = stats;
this.tableEntityConverter = new TableEntityConverter();
this.taskHubName = settings.TaskHubName;
CloudStorageAccount account = CloudStorageAccount.Parse(settings.StorageConnectionString);
this.storageAccountName = account.Credentials.AccountName;
CloudTableClient tableClient = account.CreateCloudTableClient();
tableClient.BufferManager = SimpleBufferManager.Shared;
string historyTableName = $"{taskHubName}History";
NameValidator.ValidateTableName(historyTableName);
string instancesTableName = $"{taskHubName}Instances";
NameValidator.ValidateTableName(instancesTableName);
this.historyTable = tableClient.GetTableReference(historyTableName);
this.instancesTable = tableClient.GetTableReference(instancesTableName);
this.StorageTableRequestOptions = settings.HistoryTableRequestOptions;
this.eTagValues = new ConcurrentDictionary <string, string>();
// Use reflection to learn all the different event types supported by DTFx.
// This could have been hardcoded, but I generally try to avoid hardcoding of point-in-time DTFx knowledge.
Type historyEventType = typeof(HistoryEvent);
IEnumerable <Type> historyEventTypes = historyEventType.Assembly.GetTypes().Where(
t => !t.IsAbstract && t.IsSubclassOf(historyEventType));
PropertyInfo eventTypeProperty = historyEventType.GetProperty(nameof(HistoryEvent.EventType));
this.eventTypeMap = historyEventTypes.ToDictionary(
type => ((HistoryEvent)FormatterServices.GetUninitializedObject(type)).EventType);
}
public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, AzureStorageOrchestrationServiceStats stats)
{
if (!await this.blobContainer.ExistsAsync())
{
return;
}
IEnumerable <Blob> blobList = await this.blobContainer.ListBlobsAsync(sanitizedInstanceId);
var blobForDeletionTaskList = new List <Task>();
foreach (Blob blob in blobList)
{
blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync());
}
await Task.WhenAll(blobForDeletionTaskList);
}
internal async Task DeleteLargeMessageBlobs(string instanceId, AzureStorageOrchestrationServiceStats stats)
{
var blobForDeletionTaskList = new List <Task>();
if (!await this.cloudBlobContainer.ExistsAsync())
{
return;
}
CloudBlobDirectory instanceDirectory = this.cloudBlobContainer.GetDirectoryReference(instanceId);
BlobContinuationToken blobContinuationToken = null;
while (true)
{
OperationContext context = new OperationContext {
ClientRequestID = Guid.NewGuid().ToString()
};
BlobResultSegment segment = await TimeoutHandler.ExecuteWithTimeout("DeleteLargeMessageBlobs", context.ClientRequestID, cloudBlobContainer?.ServiceClient?.Credentials?.AccountName, null, () =>
{
return(instanceDirectory.ListBlobsSegmentedAsync(blobContinuationToken));
});
stats.StorageRequests.Increment();
foreach (IListBlobItem blobListItem in segment.Results)
{
var cloudBlockBlob = blobListItem as CloudBlockBlob;
CloudBlockBlob blob = this.cloudBlobContainer.GetBlockBlobReference(cloudBlockBlob?.Name);
blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync());
}
await Task.WhenAll(blobForDeletionTaskList);
stats.StorageRequests.Increment(blobForDeletionTaskList.Count);
if (blobContinuationToken == null)
{
break;
}
}
}
public static async Task <T> ExecuteWithTimeout <T>(
string operationName,
string account,
AzureStorageOrchestrationServiceSettings settings,
Func <OperationContext, CancellationToken, Task <T> > operation,
AzureStorageOrchestrationServiceStats stats = null,
string clientRequestId = null)
{
OperationContext context = new OperationContext()
{
ClientRequestID = clientRequestId ?? Guid.NewGuid().ToString()
};
if (Debugger.IsAttached)
{
// ignore long delays while debugging
return(await operation(context, CancellationToken.None));
}
while (true)
{
using (var cts = new CancellationTokenSource())
{
Task timeoutTask = Task.Delay(DefaultTimeout, cts.Token);
Task <T> operationTask = operation(context, cts.Token);
if (stats != null)
{
stats.StorageRequests.Increment();
}
Task completedTask = await Task.WhenAny(timeoutTask, operationTask);
if (Equals(timeoutTask, completedTask))
{
// If more less than DefaultTimeoutCooldown passed, increase timeouts count
// otherwise, reset the count to 1, since this is the first timeout we receive
// after a long (enough) while
if (LastTimeoutHit + DefaultTimeoutCooldown > DateTime.UtcNow)
{
NumTimeoutsHit++;
}
else
{
NumTimeoutsHit = 1;
}
LastTimeoutHit = DateTime.UtcNow;
string taskHubName = settings?.TaskHubName;
if (NumTimeoutsHit < MaxNumberOfTimeoutsBeforeRecycle)
{
string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit {NumTimeoutsHit} out of {MaxNumberOfTimeoutsBeforeRecycle} allowed timeouts. Retrying the operation.";
settings.Logger.GeneralWarning(account ?? "", taskHubName ?? "", message);
cts.Cancel();
continue;
}
else
{
string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit maximum number ({MaxNumberOfTimeoutsBeforeRecycle}) of timeouts. Terminating the process to mitigate potential deadlock.";
settings.Logger.GeneralError(account ?? "", taskHubName ?? "", message);
// Delay to ensure the ETW event gets written
await Task.Delay(TimeSpan.FromSeconds(3));
bool executeFailFast = true;
Task <bool> gracefulShutdownTask = Task.Run(async() =>
{
try
{
return(await settings.OnImminentFailFast(message));
}
catch (Exception)
{
return(true);
}
});
await Task.WhenAny(gracefulShutdownTask, Task.Delay(TimeSpan.FromSeconds(35)));
if (gracefulShutdownTask.IsCompleted)
{
executeFailFast = gracefulShutdownTask.Result;
}
if (executeFailFast)
{
TimeoutHandler.ProcessKillAction(message);
}
else
{
// Technically we don't need else as the action above would have killed the process.
// However tests don't kill the process so putting in else.
throw new TimeoutException(message);
}
}
}
cts.Cancel();
return(await operationTask);
}
}
}
private void SetupTrackingStore()
{
var stats = new AzureStorageOrchestrationServiceStats();
this.TrackingStore = new AzureTableTrackingStore(stats, this.TableMock);
}
public SafePartitionManager(
AzureStorageOrchestrationService service,
OrchestrationSessionManager sessionManager,
AzureStorageOrchestrationServiceSettings settings,
CloudStorageAccount account,
AzureStorageOrchestrationServiceStats stats)
{
this.service = service;
this.settings = settings;
this.stats = stats;
this.sessionManager = sessionManager;
string storageAccountName = account.Credentials.AccountName;
this.intentLeaseManager = new BlobLeaseManager(
settings,
settings.TaskHubName.ToLowerInvariant() + "-leases",
string.Empty,
"intent",
account.CreateCloudBlobClient(),
skipBlobContainerCreation: false,
stats);
this.intentLeaseCollectionManager = new LeaseCollectionBalancer <BlobLease>(
"intent",
settings,
storageAccountName,
this.intentLeaseManager,
new LeaseCollectionBalancerOptions
{
AcquireInterval = settings.LeaseAcquireInterval,
RenewInterval = settings.LeaseRenewInterval,
LeaseInterval = settings.LeaseInterval,
});
var currentlyOwnedIntentLeases = this.intentLeaseCollectionManager.GetCurrentlyOwnedLeases();
this.ownershipLeaseManager = new BlobLeaseManager(
settings,
settings.TaskHubName.ToLowerInvariant() + "-leases",
string.Empty,
"ownership",
account.CreateCloudBlobClient(),
skipBlobContainerCreation: false,
stats);
this.ownershipLeaseCollectionManager = new LeaseCollectionBalancer <BlobLease>(
"ownership",
this.settings,
storageAccountName,
this.ownershipLeaseManager,
new LeaseCollectionBalancerOptions
{
AcquireInterval = TimeSpan.FromSeconds(5),
RenewInterval = TimeSpan.FromSeconds(10),
LeaseInterval = TimeSpan.FromSeconds(15),
ShouldStealLeases = false
},
shouldAquireLeaseDelegate: leaseKey => currentlyOwnedIntentLeases.ContainsKey(leaseKey),
shouldRenewLeaseDelegate: leaseKey => currentlyOwnedIntentLeases.ContainsKey(leaseKey) ||
this.sessionManager.IsControlQueueReceivingMessages(leaseKey) ||
this.sessionManager.IsControlQueueProcessingMessages(leaseKey));
}