static bool ProcessMessage(QueueMessage message, int? invisibilityTimeout = null)
{
var messageProcessed = false;
Guid contextId = message.CorrelationId.HasValue ? message.CorrelationId.Value : Guid.Empty;
using (var ctx = new CorrelationContext(contextId))
{
switch (message.MessageType)
{
case MessageTypes.BeginReplicate:
messageProcessed = DoReplicateJob(message, invisibilityTimeout);
break;
case MessageTypes.ReplicateProgress:
messageProcessed = DoReplicateProgressJob(message, invisibilityTimeout);
break;
case MessageTypes.DeleteReplica:
messageProcessed = DoDeleteReplicaJob(message, invisibilityTimeout);
break;
case MessageTypes.Unknown:
default:
DashTrace.TraceWarning("Unable to process unknown message type from async queue [{0}]. Payload: {1}",
message.MessageType,
message.ToString());
// Let this message bounce around for a bit - there may be a different version running
// on another instance that knows about this message. It will be automatically discarded
// after exceeding the deque limit.
messageProcessed = false;
break;
}
}
return messageProcessed;
}
static bool DoDeleteReplicaJob(QueueMessage message, int? invisibilityTimeout = null)
{
return BlobReplicator.DeleteReplica(
message.Payload[DeleteReplicaPayload.Source],
message.Payload[DeleteReplicaPayload.Container],
message.Payload[DeleteReplicaPayload.BlobName],
message.Payload[DeleteReplicaPayload.ETag]);
}
static bool DoReplicateJob(QueueMessage message, int? invisibilityTimeout = null)
{
return BlobReplicator.BeginBlobReplication(
message.Payload[ReplicatePayload.Source],
message.Payload[ReplicatePayload.Destination],
message.Payload[ReplicatePayload.Container],
message.Payload[ReplicatePayload.BlobName],
invisibilityTimeout);
}
public void BasicQueueTest()
{
Dictionary<string, string> payload = new Dictionary<string, string>();
payload.Add(ReplicatePayload.Source, "foo");
payload.Add(ReplicatePayload.Destination, "bar");
QueueMessage message = new QueueMessage(MessageTypes.BeginReplicate, payload);
_queue.Enqueue(message);
QueueMessage fromServer = _queue.Dequeue();
Assert.AreEqual(fromServer.MessageType, MessageTypes.BeginReplicate);
var servPayload = fromServer.Payload;
Assert.AreEqual(servPayload[ReplicatePayload.Source], payload[ReplicatePayload.Source]);
Assert.AreEqual(servPayload[ReplicatePayload.Destination], payload[ReplicatePayload.Destination]);
fromServer.Delete();
}
public async Task<IHttpActionResult> UpdateConfiguration(Configuration newConfig)
{
return await DoActionAsync("ConfigurationController.UpdateConfiguration", async (serviceClient) =>
{
UpdateConfigStatus.ConfigUpdate operationStatus = null;
try
{
// Do some validation checks first
var operationStatusTask = UpdateConfigStatus.GetActiveStatus();
var serviceConfigTask = serviceClient.GetDeploymentConfiguration();
await Task.WhenAll(operationStatusTask, serviceConfigTask);
if (operationStatusTask.Result != null)
{
return BadRequest(String.Format("Operation: {0} is already underway. Wait for this operation to complete before attempting further updates.", operationStatusTask.Result.OperationId));
}
var serviceSettings = AzureServiceConfiguration.GetSettingsProjected(serviceConfigTask.Result);
if (!CompareAccountName(serviceSettings, newConfig.AccountSettings, DashConfiguration.KeyNamespaceAccount))
{
return BadRequest("Cannot change the namespace account name once is has been set.");
}
int accountIndex = 0;
foreach (var currentAccount in serviceSettings
.Where(AzureServiceConfiguration.SettingPredicateScaleoutStorage))
{
if (!CompareAccountName(currentAccount, newConfig.ScaleAccounts.Accounts.ElementAtOrDefault(accountIndex++)))
{
var removedAccount = ParseConnectionString(currentAccount.Item2);
return BadRequest(String.Format("Data account [{0]} cannot be removed.", removedAccount.AccountName));
}
}
// Do some preamble work here synchronously (begin creation of any new storage accounts - we need to return the keys),
// but then enqueue a message on our async worker queue to complete the full update in a reliable manner without having
// the client wait forever for a response. The response will be an operationid which can be checked on by calling
// the /operations/operationid endpoint.
var operationId = DashTrace.CorrelationId.ToString();
// Reconcile storage accounts - any new accounts (indicated by a blank key) we will create immediately & include the key in the returned config
var newAccounts = new[] {
ParseConnectionString(newConfig.AccountSettings, DashConfiguration.KeyNamespaceAccount, true),
ParseConnectionString(newConfig.AccountSettings, DashConfiguration.KeyDiagnosticsAccount, true)
}
.Concat(newConfig.ScaleAccounts.Accounts
.Select(account => new StorageAccountCreationInfo
{
AccountInfo = account,
}))
.Where(account => account != null && account.AccountInfo != null && String.IsNullOrWhiteSpace(account.AccountInfo.AccountKey))
.ToList();
// Start our operation log - use the namespace account specified in the configuration
var namespaceAccount = DashConfiguration.NamespaceAccount;
// See if they've specified an account in the new config
var toBeCreatedNamespace = newAccounts
.FirstOrDefault(account => String.Equals(account.ConfigKey, DashConfiguration.KeyNamespaceAccount, StringComparison.OrdinalIgnoreCase));
if (toBeCreatedNamespace == null)
{
CloudStorageAccount.TryParse(newConfig.AccountSettings[DashConfiguration.KeyNamespaceAccount], out namespaceAccount);
}
operationStatus = await UpdateConfigStatus.GetConfigUpdateStatus(operationId, namespaceAccount);
await operationStatus.UpdateStatus(UpdateConfigStatus.States.NotStarted, "Begin service update process. OperationId: [{0}]", operationId);
operationStatus.AccountsToBeCreated = newAccounts
.Select(account => account.AccountInfo.AccountName)
.ToList();
if (newAccounts.Any())
{
await operationStatus.UpdateStatus(UpdateConfigStatus.States.CreatingAccounts, "Creating new storage accounts: [{0}]", String.Join(", ", operationStatus.AccountsToBeCreated));
var newAccountTasks = await CreateStorageAccounts(serviceClient, newConfig, newAccounts);
await Task.WhenAll(newAccountTasks);
// If we just created a new account for the namespace, wire it up now
if (CloudStorageAccount.TryParse(newConfig.AccountSettings[DashConfiguration.KeyNamespaceAccount], out namespaceAccount))
{
operationStatus.StatusHandler.UpdateCloudStorageAccount(namespaceAccount);
}
// Switch AccountsToBeCreated to the request ids that the async task can verify are completed
operationStatus.AccountsToBeCreated = newAccountTasks
.Select(newAccoutTask => newAccoutTask.Result)
.Where(requestId => !String.IsNullOrWhiteSpace(requestId))
.ToList();
await operationStatus.UpdateStatus(UpdateConfigStatus.States.CreatingAccounts, "Creation of new storage accounts initiated.");
}
// TODO: When we enable storage analytics (or at least a utilization report), we will need to turn on metrics for
// every storage account here.
string asyncQueueName = newConfig.GeneralSettings[DashConfiguration.KeyWorkerQueueName];
// Work out the list of accounts to import
var newConfigSettings = newConfig.AccountSettings
.Select(setting => Tuple.Create(setting.Key, setting.Value))
.Concat(newConfig.GeneralSettings
.Select(setting => Tuple.Create(setting.Key, setting.Value)))
.Concat(newConfig.ScaleAccounts.Accounts
.Select((account, index) => Tuple.Create(String.Format("{0}{1}", DashConfiguration.KeyScaleoutAccountPrefix, index), GenerateConnectionString(account))))
.ToDictionary(setting => setting.Item1, setting => setting.Item2, StringComparer.OrdinalIgnoreCase);
var scaleoutAccounts = serviceSettings
.Where(AzureServiceConfiguration.SettingPredicateScaleoutStorage)
.Select(account => ParseConnectionString(account.Item2))
.Where(account => account != null)
.ToDictionary(account => account.AccountName, StringComparer.OrdinalIgnoreCase);
var importAccounts = newConfig.ScaleAccounts.Accounts
.Where(newAccount => !scaleoutAccounts.ContainsKey(newAccount.AccountName))
.ToList();
// Prepare the new accounts, import accounts & service configuration information into a message for the async worker.
// Despite kicking the async worker off directly here, we need the message durably enqueued so that any downstream
// failures will be retried.
var rdfeAccessToken = await GetRdfeRefreshToken();
operationStatus.AccountsToBeImported = importAccounts
.Select(account => account.AccountName)
.ToList();
var message = new QueueMessage(MessageTypes.UpdateService,
new Dictionary<string, string>
{
{ UpdateServicePayload.OperationId, operationId },
{ UpdateServicePayload.SubscriptionId, serviceClient.SubscriptionId },
{ UpdateServicePayload.ServiceName, serviceClient.ServiceName },
{ UpdateServicePayload.RefreshToken, rdfeAccessToken },
},
DashTrace.CorrelationId);
var messageWrapper = new UpdateServicePayload(message);
messageWrapper.CreateAccountRequestIds = operationStatus.AccountsToBeCreated;
messageWrapper.ImportAccounts = importAccounts
.Select(account => GenerateConnectionString(account));
messageWrapper.Settings = newConfigSettings;
// Post message to the new namespace account (it may have changed) as that is where the async workers will read from after update
await new AzureMessageQueue(namespaceAccount, asyncQueueName).EnqueueAsync(message, 0);
// Manually fire up the async worker (so that we can supply it with the new namespace account)
var queueTask = ProcessOperationMessageLoop(operationId, namespaceAccount, GenerateConnectionString(namespaceAccount), GetMessageDelay(), asyncQueueName);
newConfig.OperationId = operationId;
newConfig.ScaleAccounts.MaxAccounts = DashConfiguration.MaxDataAccounts;
return Content(HttpStatusCode.Accepted, newConfig);
}
catch (Exception ex)
{
if (operationStatus != null)
{
var task = operationStatus.UpdateStatus(UpdateConfigStatus.States.Failed, ex.ToString());
}
throw;
}
});
}
public async Task EnqueueAsync(QueueMessage payload, int? initialInvisibilityDelay = null)
{
CloudQueueMessage message = new CloudQueueMessage(payload.ToJson());
TimeSpan? invisibilityDelay = TimeSpan.FromSeconds(initialInvisibilityDelay ?? DashConfiguration.WorkerQueueInitialDelay);
await this.Queue.AddMessageAsync(message, null, invisibilityDelay, null, null);
}
public void Enqueue(QueueMessage payload, int? initialInvisibilityDelay = null)
{
EnqueueAsync(payload, initialInvisibilityDelay).Wait();
}
public AzureMessageQueue(QueueMessage sourceMessage)
{
this.Queue = ((AzureMessageItem)sourceMessage.MessageItem).ContainingQueue.Queue;
}
public static QueueMessage ConstructReplicationMessage(bool deleteReplica, string sourceAccount, string destinationAccount, string container, string blob, string destinationETag)
{
var retval = new QueueMessage(deleteReplica ? MessageTypes.DeleteReplica : MessageTypes.BeginReplicate,
new Dictionary<string, string>
{
{ ReplicatePayload.Source, deleteReplica ? destinationAccount : sourceAccount },
{ ReplicatePayload.Destination, destinationAccount },
{ ReplicatePayload.Container, container },
{ ReplicatePayload.BlobName, blob },
},
DashTrace.CorrelationId);
if (deleteReplica)
{
retval.Payload[DeleteReplicaPayload.ETag] = destinationETag;
}
return retval;
}
static void EnqueueServiceOperationUpdate(QueueMessage message, int? invisibilityTimeout = null)
{
if (message.AbandonOperation)
{
return;
}
// Ensure that this message isn't visible immediately as we'll sit in a tight loop
new AzureMessageQueue(message).Enqueue(new QueueMessage(MessageTypes.ServiceOperationUpdate,
new Dictionary<string, string>
{
{ UpdateServicePayload.OperationId, message.Payload[UpdateServicePayload.OperationId] },
{ UpdateServicePayload.SubscriptionId, message.Payload[ServiceOperationPayload.SubscriptionId] },
{ UpdateServicePayload.ServiceName, message.Payload[ServiceOperationPayload.ServiceName] },
{ UpdateServicePayload.RefreshToken, message.Payload[ServiceOperationPayload.RefreshToken] },
},
message.CorrelationId), invisibilityTimeout ?? 10);
}
static bool DoServiceOperationUpdate(QueueMessage message, int? invisibilityTimeout = null)
{
if (ServiceUpdater.UpdateOperationStatus(
message.Payload[ServiceOperationPayload.SubscriptionId],
message.Payload[ServiceOperationPayload.ServiceName],
message.Payload[ServiceOperationPayload.OperationId],
message.Payload[ServiceOperationPayload.RefreshToken],
message.AbandonOperation).Result && !message.AbandonOperation)
{
// Re-enqueue another message to continue to check on the operation
EnqueueServiceOperationUpdate(message, invisibilityTimeout);
}
return true;
}
static bool DoUpdateServiceJob(QueueMessage message, int? invisibilityTimeout = null)
{
// Extend the timeout on this message (5 mins for account creation, 1 min for account import & 30 secs to upgrade service)
message.UpdateInvisibility(invisibilityTimeout ?? 390);
var payload = new UpdateServicePayload(message);
string operationId = ServiceUpdater.ImportAccountsAndUpdateService(
message.Payload[ServiceOperationPayload.SubscriptionId],
message.Payload[ServiceOperationPayload.ServiceName],
message.Payload[ServiceOperationPayload.OperationId],
message.Payload[ServiceOperationPayload.RefreshToken],
payload.CreateAccountRequestIds,
payload.ImportAccounts,
payload.Settings,
message.AbandonOperation).Result;
if (!String.IsNullOrWhiteSpace(operationId) && !message.AbandonOperation)
{
// Enqueue a follow-up message to check the progress of this operation
EnqueueServiceOperationUpdate(message, invisibilityTimeout);
return true;
}
return false;
}
static bool DoDeleteReplicaJob(QueueMessage message, int? invisibilityTimeout = null)
{
if (message.AbandonOperation)
{
// We don't have any specific failure functionality for this operation
return true;
}
return BlobReplicator.DeleteReplica(
message.Payload[DeleteReplicaPayload.Source],
message.Payload[DeleteReplicaPayload.Container],
message.Payload[DeleteReplicaPayload.BlobName],
message.Payload[DeleteReplicaPayload.ETag]);
}
static bool DoReplicateProgressJob(QueueMessage message, int? invisibilityTimeout = null)
{
if (message.AbandonOperation)
{
// We don't have any specific failure functionality for this operation
return true;
}
return BlobReplicator.ProgressBlobReplication(
message.Payload[ReplicateProgressPayload.Source],
message.Payload[ReplicateProgressPayload.Destination],
message.Payload[ReplicateProgressPayload.Container],
message.Payload[ReplicateProgressPayload.BlobName],
message.Payload[ReplicateProgressPayload.CopyID],
invisibilityTimeout);
}
public void Enqueue(QueueMessage payload, int?initialInvisibilityDelay = null)
{
EnqueueAsync(payload, initialInvisibilityDelay).Wait();
}