private async Task <AzureQueueDataManager> GetTableManager(string qName, TimeSpan?visibilityTimeout = null)
{
AzureQueueDataManager manager = new AzureQueueDataManager(this.loggerFactory, qName, DeploymentId, TestDefaultConfiguration.DataConnectionString, visibilityTimeout);
await manager.InitQueueAsync();
return(manager);
}
private async Task <AzureQueueDataManager> GetTableManager(string qName)
{
AzureQueueDataManager manager = new AzureQueueDataManager(qName, DeploymentId, TestDefaultConfiguration.DataConnectionString);
await manager.InitQueueAsync();
return(manager);
}
public async Task AQ_Standalone_4()
{
TimeSpan visibilityTimeout = TimeSpan.FromSeconds(2);
queueName = "Test-5-".ToLower() + Guid.NewGuid();
AzureQueueDataManager manager = await GetTableManager(queueName, visibilityTimeout);
Assert.Equal(0, await manager.GetApproximateMessageCount());
var inMessage = "Hello, World";
await manager.AddQueueMessage(inMessage);
Assert.Equal(1, await manager.GetApproximateMessageCount());
QueueMessage outMessage = await manager.GetQueueMessage();
logger.Info("GetQueueMessage: {0}", PrintQueueMessage(outMessage));
Assert.Equal(inMessage, outMessage.MessageText);
await Task.Delay(visibilityTimeout);
Assert.Equal(1, await manager.GetApproximateMessageCount());
QueueMessage outMessage2 = await manager.GetQueueMessage();
Assert.Equal(inMessage, outMessage2.MessageText);
await manager.DeleteQueueMessage(outMessage2);
Assert.Equal(0, await manager.GetApproximateMessageCount());
}
private async Task <AzureQueueDataManager> GetTableManager(string qName)
{
AzureQueueDataManager manager = new AzureQueueDataManager(qName, DeploymentId, StorageTestConstants.DataConnectionString);
await manager.InitQueueAsync();
return(manager);
}
public static IQueueAdapterReceiver Create(SerializationManager serializationManager, QueueId queueId, string dataConnectionString, string deploymentId, IAzureQueueDataAdapter dataAdapter, TimeSpan?messageVisibilityTimeout = null)
{
if (queueId == null)
{
throw new ArgumentNullException(nameof(queueId));
}
if (string.IsNullOrEmpty(dataConnectionString))
{
throw new ArgumentNullException(nameof(dataConnectionString));
}
if (string.IsNullOrEmpty(deploymentId))
{
throw new ArgumentNullException(nameof(deploymentId));
}
if (dataAdapter == null)
{
throw new ArgumentNullException(nameof(dataAdapter));
}
if (serializationManager == null)
{
throw new ArgumentNullException(nameof(serializationManager));
}
var queue = new AzureQueueDataManager(queueId.ToString(), deploymentId, dataConnectionString, messageVisibilityTimeout);
return(new AzureQueueAdapterReceiver(serializationManager, queueId, queue, dataAdapter));
}
public async Task AQ_Standalone_2()
{
queueName = "Test-2-".ToLower() + Guid.NewGuid();
AzureQueueDataManager manager = await GetTableManager(queueName);
IEnumerable <CloudQueueMessage> msgs = await manager.GetQueueMessages();
Assert.IsTrue(msgs == null || msgs.Count() == 0);
int numMsgs = 10;
List <Task> promises = new List <Task>();
for (int i = 0; i < numMsgs; i++)
{
promises.Add(manager.AddQueueMessage(new CloudQueueMessage(i.ToString())));
}
Task.WaitAll(promises.ToArray());
Assert.AreEqual(numMsgs, await manager.GetApproximateMessageCount());
msgs = new List <CloudQueueMessage>(await manager.GetQueueMessages(numMsgs));
Assert.AreEqual(numMsgs, msgs.Count());
Assert.AreEqual(numMsgs, await manager.GetApproximateMessageCount());
promises = new List <Task>();
foreach (var msg in msgs)
{
promises.Add(manager.DeleteQueueMessage(msg));
}
Task.WaitAll(promises.ToArray());
Assert.AreEqual(0, await manager.GetApproximateMessageCount());
}
private AzureQueueAdapterReceiver(string azureQueueName, ILoggerFactory loggerFactory, AzureQueueDataManager queue, IQueueDataAdapter <string, IBatchContainer> dataAdapter)
{
this.azureQueueName = azureQueueName ?? throw new ArgumentNullException(nameof(azureQueueName));
this.queue = queue ?? throw new ArgumentNullException(nameof(queue));
this.dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter));
this.logger = loggerFactory.CreateLogger <AzureQueueAdapterReceiver>();
this.pending = new List <PendingDelivery>();
}
private async Task <AzureQueueDataManager> GetTableManager(string qName, TimeSpan?visibilityTimeout = null)
{
AzureQueueDataManager manager = new AzureQueueDataManager(this.loggerFactory, $"{qName}-{DeploymentId}", new AzureQueueOptions {
MessageVisibilityTimeout = visibilityTimeout
}.ConfigureTestDefaults());
await manager.InitQueueAsync();
return(manager);
}
private AzureQueueAdapterReceiver(SerializationManager serializationManager, string azureQueueName, ILoggerFactory loggerFactory, AzureQueueDataManager queue, IAzureQueueDataAdapter dataAdapter)
{
this.azureQueueName = azureQueueName ?? throw new ArgumentNullException(nameof(azureQueueName));
this.serializationManager = serializationManager;
this.queue = queue ?? throw new ArgumentNullException(nameof(queue));
this.dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter));
this.logger = loggerFactory.CreateLogger <AzureQueueAdapterReceiver>();
this.pending = new List <PendingDelivery>();
}
public async Task QueueMessageBatchAsync <T>(Guid streamGuid, String streamNamespace, IEnumerable <T> events, StreamSequenceToken token, Dictionary <string, object> requestContext)
{
if (events == null)
{
throw new ArgumentNullException("events", "Trying to QueueMessageBatchAsync null data.");
}
//int count = events.Count();
//if (count != 1)
//{
// throw new OrleansException("Trying to QueueMessageBatchAsync a batch of more than one event. " +
// "SimpleAzureQueueAdapter does not support batching. Instead, you can batch in your application code.");
//}
object data = events.First();
bool isBytes = data is byte[];
bool isString = data is string;
if (data != null && !isBytes && !isString)
{
throw new OrleansException(
string.Format(
"Trying to QueueMessageBatchAsync a type {0} which is not a byte[] and not string. " +
"SimpleAzureQueueAdapter only supports byte[] or string.", data.GetType()));
}
if (Queue == null)
{
var tmpQueue = new AzureQueueDataManager(this.loggerFactory, QueueName, DataConnectionString);
await tmpQueue.InitQueueAsync();
if (Queue == null)
{
Queue = tmpQueue;
}
}
CloudQueueMessage cloudMsg = null;
if (isBytes)
{
//new CloudQueueMessage(byte[]) not supported in netstandard
cloudMsg = new CloudQueueMessage(null as string);
cloudMsg.SetMessageContent2(data as byte[]);
}
else if (isString)
{
cloudMsg = new CloudQueueMessage(data as string);
}
else if (data == null)
{
// It's OK to pass null data. why should I care?
cloudMsg = new CloudQueueMessage(null as string);
}
await Queue.AddQueueMessage(cloudMsg);
}
/// <summary>
/// Helper method for testing. Clears all messages in all the queues used by the specified stream provider.
/// </summary>
/// <param name="loggerFactory">logger factory to use</param>
/// <param name="azureQueueNames">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task ClearAllUsedAzureQueues(ILoggerFactory loggerFactory, List <string> azureQueueNames, string storageConnectionString)
{
var deleteTasks = new List <Task>();
foreach (var queueName in azureQueueNames)
{
var manager = new AzureQueueDataManager(loggerFactory, queueName, storageConnectionString);
deleteTasks.Add(manager.ClearQueue());
}
await Task.WhenAll(deleteTasks);
}
/// <summary>
/// Helper method for testing. Deletes all the queues used by the specified stream provider.
/// </summary>
/// <param name="loggerFactory">logger factory to use</param>
/// <param name="azureQueueNames">azure queue names to be deleted.</param>
/// <param name="queueOptions">The azure storage options.</param>
public static async Task DeleteAllUsedAzureQueues(ILoggerFactory loggerFactory, List <string> azureQueueNames, AzureQueueOptions queueOptions)
{
var deleteTasks = new List <Task>();
foreach (var queueName in azureQueueNames)
{
var manager = new AzureQueueDataManager(loggerFactory, queueName, queueOptions);
deleteTasks.Add(manager.DeleteQueue());
}
await Task.WhenAll(deleteTasks);
}
private AzureQueueAdapterReceiver(QueueId queueId, AzureQueueDataManager queue)
{
if (queueId == null)
{
throw new ArgumentNullException("queueId");
}
if (queue == null)
{
throw new ArgumentNullException("queue");
}
Id = queueId;
this.queue = queue;
}
private AzureQueueAdapterReceiver(QueueId queueId, AzureQueueDataManager queue)
{
if (queueId == null)
{
throw new ArgumentNullException("queueId");
}
if (queue == null)
{
throw new ArgumentNullException("queue");
}
Id = queueId;
this.queue = queue;
logger = LogManager.GetLogger(GetType().Name, LoggerType.Provider);
}
private AzureQueueAdapterReceiver(QueueId queueId, AzureQueueDataManager queue, int cacheSize)
{
if (queueId == null)
{
throw new ArgumentNullException("queueId");
}
if (queue == null)
{
throw new ArgumentNullException("queue");
}
Id = queueId;
this.queue = queue;
cache = new SimpleQueueAdapterCache(cacheSize);
}
public async Task QueueMessageBatchAsync <T>(Guid streamGuid, String streamNamespace, IEnumerable <T> events)
{
var queueId = streamQueueMapper.GetQueueForStream(streamGuid);
AzureQueueDataManager queue;
if (!Queues.TryGetValue(queueId, out queue))
{
var tmpQueue = new AzureQueueDataManager(queueId.ToString(), DeploymentId, DataConnectionString);
await tmpQueue.InitQueueAsync();
queue = Queues.GetOrAdd(queueId, tmpQueue);
}
var cloudMsg = AzureQueueBatchContainer.ToCloudQueueMessage(streamGuid, streamNamespace, events);
await queue.AddQueueMessage(cloudMsg);
}
public async Task Shutdown(TimeSpan timeout)
{
try
{
// await the last storage operation, so after we shutdown and stop this receiver we don't get async operation completions from pending storage operations.
if (outstandingTask != null)
{
await outstandingTask;
}
}
finally
{
// remember that we shut down so we never try to read from the queue again.
queue = null;
}
}
/// <summary>
/// Helper method for testing. Deletes all the queues used by the specifed stream provider.
/// </summary>
/// <param name="providerName">The Azure Queue stream privider name.</param>
/// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task DeleteAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.DEFAULT_NUM_QUEUES, providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
deleteTasks.Add(manager.DeleteQueue());
}
await Task.WhenAll(deleteTasks);
}
}
/// <summary>
/// Helper method for testing. Clears all messages in all the queues used by the specifed stream provider.
/// </summary>
/// <param name="providerName">The Azure Queue stream privider name.</param>
/// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task ClearAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterConstants.NumQueuesDefaultValue, providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
deleteTasks.Add(manager.ClearQueue());
}
await Task.WhenAll(deleteTasks);
}
}
public async Task QueueMessageBatchAsync <T>(StreamId streamId, IEnumerable <T> events, StreamSequenceToken token, Dictionary <string, object> requestContext)
{
if (events == null)
{
throw new ArgumentNullException("events", "Trying to QueueMessageBatchAsync null data.");
}
object data = events.First();
bool isBytes = data is byte[];
bool isString = data is string;
if (data != null && !isBytes && !isString)
{
throw new OrleansException(
string.Format(
"Trying to QueueMessageBatchAsync a type {0} which is not a byte[] and not string. " +
"SimpleAzureQueueAdapter only supports byte[] or string.", data.GetType()));
}
if (Queue == null)
{
var tmpQueue = new AzureQueueDataManager(this.loggerFactory, options.QueueName,
new AzureQueueOptions {
ConnectionString = options.ConnectionString, ServiceUri = options.ServiceUri, TokenCredential = options.TokenCredential
});
await tmpQueue.InitQueueAsync();
if (Queue == null)
{
Queue = tmpQueue;
}
}
string cloudMsg = null;
if (isBytes)
{
cloudMsg = Convert.ToBase64String(data as byte[]);
}
else if (isString)
{
cloudMsg = data as string;
}
await Queue.AddQueueMessage(cloudMsg);
}
/// <summary>
/// Helper method for testing. Clears all messages in all the queues used by the specifed stream provider.
/// </summary>
/// <param name="loggerFactory">logger factory to use</param>
/// <param name="providerName">The Azure Queue stream privider name.</param>
/// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task ClearAllUsedAzureQueues(ILoggerFactory loggerFactory, string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
// TODO: Do not assume defaults !? - jbragg
var queueMapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions(), providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List <Task>();
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(loggerFactory, queueId.ToString(), deploymentId, storageConnectionString);
deleteTasks.Add(manager.ClearQueue());
}
await Task.WhenAll(deleteTasks);
}
}
/// <summary>
/// Helper method for testing.
/// </summary>
/// <param name="providerName"></param>
/// <param name="deploymentId"></param>
/// <param name="storageConnectionString"></param>
/// <param name="logger"></param>
/// <returns></returns>
public static async Task DeleteAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString, Logger logger)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.DEFAULT_NUM_QUEUES, providerName);
List <QueueId> allQueues = queueMapper.GetAllQueues().ToList();
if (logger != null)
{
logger.Info("About to delete all {0} Stream Queues\n", allQueues.Count);
}
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
await manager.DeleteQueue();
}
}
}
public async Task AQ_Standalone_3_Init_MultipleThreads()
{
queueName = "Test-4-".ToLower() + Guid.NewGuid();
const int NumThreads = 100;
Task <bool>[] promises = new Task <bool> [NumThreads];
for (int i = 0; i < NumThreads; i++)
{
promises[i] = Task.Run <bool>(async() =>
{
AzureQueueDataManager manager = await GetTableManager(queueName);
return(true);
});
}
await Task.WhenAll(promises);
}
public static IQueueAdapterReceiver Create(QueueId queueId, string dataConnectionString, string deploymentId)
{
if (queueId == null)
{
throw new ArgumentNullException("queueId");
}
if (String.IsNullOrEmpty(dataConnectionString))
{
throw new ArgumentNullException("dataConnectionString");
}
if (String.IsNullOrEmpty(deploymentId))
{
throw new ArgumentNullException("deploymentId");
}
var queue = new AzureQueueDataManager(queueId.ToString(), deploymentId, dataConnectionString);
return(new AzureQueueAdapterReceiver(queueId, queue));
}
public async Task QueueMessageBatchAsync <T>(Guid streamGuid, string streamNamespace, IEnumerable <T> events, StreamSequenceToken token, Dictionary <string, object> requestContext)
{
if (token != null)
{
throw new ArgumentException("AzureQueue stream provider currently does not support non-null StreamSequenceToken.", nameof(token));
}
var queueId = streamQueueMapper.GetQueueForStream(streamGuid, streamNamespace);
AzureQueueDataManager queue;
if (!Queues.TryGetValue(queueId, out queue))
{
var tmpQueue = new AzureQueueDataManager(this.loggerFactory, queueId.ToString(), ServiceId, DataConnectionString, MessageVisibilityTimeout);
await tmpQueue.InitQueueAsync();
queue = Queues.GetOrAdd(queueId, tmpQueue);
}
var cloudMsg = this.dataAdapter.ToCloudQueueMessage(streamGuid, streamNamespace, events, requestContext);
await queue.AddQueueMessage(cloudMsg);
}
public static IQueueAdapterReceiver Create(ILoggerFactory loggerFactory, string azureQueueName, AzureQueueOptions queueOptions, IQueueDataAdapter <string, IBatchContainer> dataAdapter)
{
if (azureQueueName == null)
{
throw new ArgumentNullException(nameof(azureQueueName));
}
if (queueOptions == null)
{
throw new ArgumentNullException(nameof(queueOptions));
}
if (dataAdapter == null)
{
throw new ArgumentNullException(nameof(dataAdapter));
}
var queue = new AzureQueueDataManager(loggerFactory, azureQueueName, queueOptions);
return(new AzureQueueAdapterReceiver(azureQueueName, loggerFactory, queue, dataAdapter));
}
public async Task QueueMessageBatchAsync <T>(StreamId streamId, IEnumerable <T> events, StreamSequenceToken token, Dictionary <string, object> requestContext)
{
if (token != null)
{
throw new ArgumentException("AzureQueue stream provider currently does not support non-null StreamSequenceToken.", nameof(token));
}
var queueId = streamQueueMapper.GetQueueForStream(streamId);
AzureQueueDataManager queue;
if (!Queues.TryGetValue(queueId, out queue))
{
var tmpQueue = new AzureQueueDataManager(this.loggerFactory, this.streamQueueMapper.PartitionToAzureQueue(queueId), queueOptions);
await tmpQueue.InitQueueAsync();
queue = Queues.GetOrAdd(queueId, tmpQueue);
}
var cloudMsg = this.dataAdapter.ToQueueMessage(streamId, events, null, requestContext);
await queue.AddQueueMessage(cloudMsg);
}
public async Task QueueMessageBatchAsync <T>(Guid streamGuid, String streamNamespace, IEnumerable <T> events, StreamSequenceToken token, Dictionary <string, object> requestContext)
{
if (token != null)
{
throw new ArgumentException("AzureQueue stream provider currebtly does not support non-null StreamSequenceToken.", "token");
}
var queueId = streamQueueMapper.GetQueueForStream(streamGuid, streamNamespace);
AzureQueueDataManager queue;
if (!Queues.TryGetValue(queueId, out queue))
{
var tmpQueue = new AzureQueueDataManager(queueId.ToString(), DeploymentId, DataConnectionString);
await tmpQueue.InitQueueAsync();
queue = Queues.GetOrAdd(queueId, tmpQueue);
}
var cloudMsg = AzureQueueBatchContainer.ToCloudQueueMessage(streamGuid, streamNamespace, events, requestContext);
await queue.AddQueueMessage(cloudMsg);
}
private AzureQueueAdapterReceiver(QueueId queueId, AzureQueueDataManager queue, IAzureQueueDataAdapter dataAdapter)
{
if (queueId == null)
{
throw new ArgumentNullException(nameof(queueId));
}
if (queue == null)
{
throw new ArgumentNullException(nameof(queue));
}
if (dataAdapter == null)
{
throw new ArgumentNullException(nameof(queue));
}
Id = queueId;
this.queue = queue;
this.dataAdapter = dataAdapter;
this.logger = LogManager.GetLogger(GetType().Name, LoggerType.Provider);
this.pending = new List <PendingDelivery>();
}
public async Task AQ_Standalone_1()
{
queueName = "Test-1-".ToLower() + Guid.NewGuid();
AzureQueueDataManager manager = await GetTableManager(queueName);
Assert.AreEqual(0, await manager.GetApproximateMessageCount());
CloudQueueMessage inMessage = new CloudQueueMessage("Hello, World");
await manager.AddQueueMessage(inMessage);
//Nullable<int> count = manager.ApproximateMessageCount;
Assert.AreEqual(1, await manager.GetApproximateMessageCount());
CloudQueueMessage outMessage1 = await manager.PeekQueueMessage();
logger.Info("PeekQueueMessage 1: {0}", AzureStorageUtils.PrintCloudQueueMessage(outMessage1));
Assert.AreEqual(inMessage.AsString, outMessage1.AsString);
CloudQueueMessage outMessage2 = await manager.PeekQueueMessage();
logger.Info("PeekQueueMessage 2: {0}", AzureStorageUtils.PrintCloudQueueMessage(outMessage2));
Assert.AreEqual(inMessage.AsString, outMessage2.AsString);
CloudQueueMessage outMessage3 = await manager.GetQueueMessage();
logger.Info("GetQueueMessage 3: {0}", AzureStorageUtils.PrintCloudQueueMessage(outMessage3));
Assert.AreEqual(inMessage.AsString, outMessage3.AsString);
Assert.AreEqual(1, await manager.GetApproximateMessageCount());
CloudQueueMessage outMessage4 = await manager.GetQueueMessage();
Assert.IsNull(outMessage4);
Assert.AreEqual(1, await manager.GetApproximateMessageCount());
await manager.DeleteQueueMessage(outMessage3);
Assert.AreEqual(0, await manager.GetApproximateMessageCount());
}