public static void ProcessMessageLoop(ref int msgProcessed, ref int msgErrors, int? invisibilityTimeout = null)
{
IMessageQueue queue = new AzureMessageQueue();
while (true)
{
try
{
QueueMessage payload = queue.Dequeue(invisibilityTimeout);
if (payload == null)
{
break;
}
// Right now, success/failure is indicated through a bool
// Do we want to surround this with a try/catch and use exceptions instead?
if (ProcessMessage(payload, invisibilityTimeout))
{
queue.DeleteCurrentMessage();
msgProcessed++;
}
else
{
// Leave it in the queue for retry after invisibility period expires
msgErrors++;
}
}
catch (Exception ex)
{
DashTrace.TraceWarning("Unhandled exception processing async message. Message will be left in queue. Details: {0}", ex);
msgErrors++;
}
}
}
public static void Init(TestContext ctx)
{
_ctx = InitializeConfig(ctx, "datax2", new Dictionary<string, string>
{
{ "AccountName", "dashtest" },
});
_queue = new AzureMessageQueue(null, Guid.NewGuid().ToString());
}
public static void ProcessMessageLoop(ref int msgProcessed,
ref int msgErrors,
Func<QueueMessage, bool> continueLoopPostDequeue,
Func<bool> continueLoopPostProcess,
int? invisibilityTimeout,
string namespaceAccount,
string queueName)
{
CloudStorageAccount newNamespace = null;
if (!String.IsNullOrWhiteSpace(namespaceAccount))
{
if (CloudStorageAccount.TryParse(namespaceAccount, out newNamespace))
{
DashConfiguration.NamespaceAccount = newNamespace;
}
}
if (DashConfiguration.NamespaceAccount == null)
{
DashTrace.TraceWarning("Namespace account configuration not set. Asynchronous processing cannot continue.");
return;
}
IMessageQueue queue = new AzureMessageQueue(newNamespace, queueName);
do
{
try
{
QueueMessage payload = queue.Dequeue(invisibilityTimeout);
if (!continueLoopPostDequeue(payload))
{
break;
}
// Right now, success/failure is indicated through a bool
// Do we want to surround this with a try/catch and use exceptions instead?
if (payload != null)
{
if (ProcessMessage(payload, invisibilityTimeout) || payload.AbandonOperation)
{
payload.Delete();
msgProcessed++;
}
else
{
// Leave it in the queue for retry after invisibility period expires
msgErrors++;
}
}
}
catch (Exception ex)
{
DashTrace.TraceWarning("Unhandled exception processing async message. Message will be left in queue. Details: {0}", ex);
msgErrors++;
}
}
while (continueLoopPostProcess());
}
public void Init()
{
_runner = new WebApiTestRunner(new Dictionary<string, string>()
{
{ "AccountName", "dashtest" },
{ "StorageConnectionStringMaster", "DefaultEndpointsProtocol=https;AccountName=dashtestnamespace;AccountKey=N+BMOAp/bswfqp4dxoQYLLwmYnERysm1Xxv3qSf5H9RVhQ0q+f/QKNHhXX4Z/P67mZ+5QwT6RZv9qKV834pOqQ==" },
{ "ScaleoutStorage0", "DefaultEndpointsProtocol=https;AccountName=dashtestdata1;AccountKey=IatOQyIdf8x3HcCZuhtGGLv/nS0v/SwXu2vBS6E9/5/+GYllhdmFFX6YqMXmR7U6UyFYQt4pdZnlLCM+bPcJ4A==" },
{ "ScaleoutStorage1", "DefaultEndpointsProtocol=https;AccountName=dashtestdata2;AccountKey=OOXSVWWpImRf79sbiEtpIwFsggv7VAhdjtKdt7o0gOLr2krzVXwZ+cb/gJeMqZRlXHTniRN6vnKKjs1glijihA==" },
{ "ScaleoutNumberOfAccounts", "2"},
});
_queue = new AzureMessageQueue(Guid.NewGuid().ToString());
}
bool IsWorkerQueueEmpty()
{
var queue = new AzureMessageQueue();
var nextMessage = queue.Queue.PeekMessage();
if (nextMessage == null)
{
Task.Delay(1000).Wait();
nextMessage = queue.Queue.PeekMessage();
return nextMessage == null;
}
return false;
}
public static async Task EnqueueBlobReplicationAsync(NamespaceBlob namespaceBlob, bool deleteReplica, bool saveNamespaceEntry = true)
{
if (!await namespaceBlob.ExistsAsync())
{
return;
}
// Trim down the namespace replication list to the first 'master' item. This is sufficient to ensure that the
// orphaned blobs are not effectively in the account. The master blob will be replicated over the top of the
// orphaned blobs.
string primaryAccount = namespaceBlob.PrimaryAccountName;
if (namespaceBlob.IsReplicated)
{
namespaceBlob.PrimaryAccountName = primaryAccount;
if (saveNamespaceEntry)
{
await namespaceBlob.SaveAsync();
}
}
// This rest of this method does not block. Enqueueing the replication is a completely async process
var task = Task.Factory.StartNew(() =>
{
var queue = new AzureMessageQueue();
var tasks = DashConfiguration.DataAccounts
.Where(dataAccount => !dataAccount.Credentials.AccountName.Equals(primaryAccount, StringComparison.OrdinalIgnoreCase))
.Select(async dataAccount => await queue.EnqueueAsync(ConstructReplicationMessage(deleteReplica,
primaryAccount,
dataAccount.Credentials.AccountName,
namespaceBlob.Container,
namespaceBlob.BlobName,
deleteReplica ? await GetBlobETagAsync(dataAccount, namespaceBlob.Container, namespaceBlob.BlobName) : null)));
Task.WhenAll(tasks)
.ContinueWith(antecedent =>
{
if (antecedent.Exception != null)
{
DashTrace.TraceWarning("Error queueing replication message for blob: {0}. Details: {1}",
PathUtils.CombineContainerAndBlob(namespaceBlob.Container, namespaceBlob.BlobName),
antecedent.Exception.Flatten());
}
else
{
DashTrace.TraceInformation("Blob: {0} has been enqueued for replication.",
PathUtils.CombineContainerAndBlob(namespaceBlob.Container, namespaceBlob.BlobName));
}
});
});
}
void AssertReplicationMessageIsEnqueued(MessageTypes messageType, string container, string blobName, string primaryAccount)
{
// Wait for the messages to be fully enqueued
Task.Delay(1000).Wait();
var queue = new AzureMessageQueue();
var replicaAccounts = DashConfiguration.DataAccounts
.Select(account => account.Credentials.AccountName)
.Where(accountName => !String.Equals(accountName, primaryAccount, StringComparison.OrdinalIgnoreCase))
.ToDictionary(accountName => accountName, accountName => false, StringComparer.OrdinalIgnoreCase);
while (true)
{
var replicateMessage = queue.Dequeue();
if (replicateMessage == null)
{
break;
}
Assert.IsNotNull(replicateMessage);
Assert.AreEqual(replicateMessage.MessageType, messageType);
if (messageType == MessageTypes.BeginReplicate)
{
Assert.AreEqual(replicateMessage.Payload[ReplicatePayload.Source], primaryAccount);
}
Assert.AreEqual(replicateMessage.Payload[ReplicatePayload.Container], container);
Assert.AreEqual(replicateMessage.Payload[ReplicatePayload.BlobName], blobName);
replicaAccounts[replicateMessage.Payload[messageType == MessageTypes.BeginReplicate ? ReplicatePayload.Destination : ReplicatePayload.Source]] = true;
queue.DeleteCurrentMessage();
}
Assert.IsFalse(replicaAccounts.Any(account => !account.Value),
"Data accounts detected with no replication enqueued: {0}",
String.Join(", ", replicaAccounts
.Where(account => !account.Value)
.Select(account => account.Key)));
}
public void Cleanup()
{
var queue = new AzureMessageQueue();
queue.DeleteQueue();
}
void AssertQueueIsDrained()
{
// Wait for the messages to be fully enqueued
Task.Delay(1000).Wait();
bool messageSeen = false;
var queue = new AzureMessageQueue();
while (true)
{
var message = queue.Dequeue();
if (message == null)
{
break;
}
queue.DeleteCurrentMessage();
messageSeen = true;
}
if (messageSeen)
{
Assert.Fail("Expected queue to be empty");
}
}