/// <summary>
/// Get a request options
/// </summary>
/// <param name="type">Service type</param>
/// <returns>Request options</returns>
public IRequestOptions GetRequestOptions(StorageServiceType type)
{
IRequestOptions options;
switch (type)
{
case StorageServiceType.Blob:
options = new BlobRequestOptions();
break;
case StorageServiceType.Queue:
options = new QueueRequestOptions();
break;
case StorageServiceType.Table:
options = new TableRequestOptions();
break;
case StorageServiceType.File:
options = new FileRequestOptions();
break;
default:
throw new ArgumentException(Resources.InvalidStorageServiceType, "type");
}
if (this.ServerTimeoutPerRequest.HasValue)
{
options.ServerTimeout = ConvertToTimeSpan(this.ServerTimeoutPerRequest.Value);
}
if (this.ClientTimeoutPerRequest.HasValue)
{
options.MaximumExecutionTime = ConvertToTimeSpan(this.ClientTimeoutPerRequest.Value);
}
return(options);
}
static void Main(string[] args)
{
var account = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
var options = new QueueRequestOptions {
RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(2), 10)
};
var queueClient = account.CreateCloudQueueClient();
var queue = queueClient.GetQueueReference("orders");
queue.CreateIfNotExists();
while (true)
{
var message = queue.GetMessage(TimeSpan.FromSeconds(60), options);
if (null == message)
{
Console.WriteLine("No orders found. Hitting the snooze button...");
Thread.Sleep(5000);
continue;
}
using (var messageHeartbeatTimer = KeepMessageHidden(queue, message, options, 45000))
{
try
{
Console.Write("Processing order {0}...", message.AsString);
Thread.Sleep(500);
Console.WriteLine("Complete.");
queue.DeleteMessage(message, options);
}
finally
{
messageHeartbeatTimer?.Stop();
}
}
}
}
public ContextAndOptions()
{
Context = new OperationContext()
{
ClientRequestID = $"my id {DateTime.UtcNow.Millisecond}",
LogLevel = LogLevel.Verbose
};
Context.RequestCompleted += (o, requestEventArgs) =>
{
Console.WriteLine($"Request to queue completed.Client Id='" +
$"{(o as OperationContext).ClientRequestID}' " +
$"ServerId='{requestEventArgs.RequestInformation.ServiceRequestID}'");
};
Options = new QueueRequestOptions()
{
LocationMode = LocationMode.PrimaryThenSecondary,
RetryPolicy = new LinearRetry(),
MaximumExecutionTime = TimeSpan.FromSeconds(5),
ServerTimeout = TimeSpan.FromSeconds(2)
};
}
/// <summary>
/// Queueへの参照
/// </summary>
/// <param name="storageAccount"></param>
/// <param name="queueName"></param>
/// <returns></returns>
private static CloudQueue GetQueueReference(CloudStorageAccount storageAccount, string queueName)
{
// queue client を作成
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// 再試行ポリシーの構成
QueueRequestOptions interactiveRequestOption = new QueueRequestOptions()
{
RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(1), 5),
//geo 冗長ストレージ(GRS)の場合、PrimaryThenSecondaryを設定する
//それ以外は、PrimaryOnlyを設定する
LocationMode = LocationMode.PrimaryOnly,
MaximumExecutionTime = TimeSpan.FromSeconds(10)
};
queueClient.DefaultRequestOptions = interactiveRequestOption;
// queue名に大文字は使えないので小文字に変換する
queueName = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToLower(queueName);
// queueへの参照を取得する
return(queueClient.GetQueueReference(queueName));
}
/// <summary>
/// Will serialize the data and add it to a queue with the same name as the class object
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="data"></param>
public static void AddQueueItem <T>(T data, string queueName)
{
//var queueName = typeof(T).Name;
var queue = GetQueueReference(queueName);
var dataString = JsonConvert.SerializeObject(data);
if (dataString.Length > 65535)
{
var blobContainer = GetContainer(queueName);
var filename = Guid.NewGuid().ToString() + ".json";
dataString = AddBlobText(blobContainer, filename, dataString);
}
CloudQueueMessage message = new CloudQueueMessage(dataString);
IRetryPolicy linearRetryPolicy = new LinearRetry(TimeSpan.FromSeconds(5400), 3);
var options = new QueueRequestOptions {
RetryPolicy = linearRetryPolicy
};
queue.AddMessage(message, null, new TimeSpan(0, 1, 0), options, null);
}
/// <summary>
/// List storage queues
/// </summary>
/// <param name="prefix">Queue name prefix</param>
/// <param name="queueListingDetails">Queue listing details</param>
/// <param name="options">Queue request options</param>
/// <param name="operationContext">Operation context</param>
/// <returns>An enumerable collection of the queues in the storage account.</returns>
public IEnumerable <CloudQueue> ListQueues(string prefix, QueueListingDetails queueListingDetails,
QueueRequestOptions options, OperationContext operationContext)
{
return(queueClient.ListQueues(prefix, queueListingDetails, options, operationContext));
}
/// <summary>
/// Fetch queue attributes
/// </summary>
/// <param name="queue">Cloud queue object</param>
/// <param name="options">Queue request options</param>
/// <param name="operationContext">Operation context</param>
public void FetchAttributes(CloudQueue queue, QueueRequestOptions options, OperationContext operationContext)
{
queue.FetchAttributes(options, operationContext);
}
/* For the time being, we don't support updating the content of a message due to complexity
* In order to support updating content we need to consider the following scenarios
* 1) Previous content was smaller than max size and we are updating with content that is also smaller than max size. This is a trivial scenario. We simply need to update the content in the Azure queue.
* 2) Previous content exceeded max size and we are updating with content that also exceeds max size. This is also a trivial scenario. We simply need to update the content in the blob.
* 3) Previous content was smaller than max size and we are updating with content that exceeds max size. We need to save the new content in a blob, and the queue message must be updated with a 'LargeMessageEnvelope'
* 4) Previous content exceeded max size and we are updating with content smaller than max size. We need to delete the blob item and update the queue message.
*
* Determining if the new content exceeds max size or not is easy (see AddMessageAsync) but how can we determine if previous content exceeded the max size?
*
* public Task UpdateMessageAsync(CloudMessage message, TimeSpan visibilityTimeout, MessageUpdateFields updateFields, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
* {
* var cloudMessage = new CloudQueueMessage(message.Id, message.PopReceipt);
* return _queue.UpdateMessageAsync(cloudMessage, visibilityTimeout, updateFields, options, operationContext, cancellationToken);
* }
*/
public Task UpdateMessageVisibilityTimeoutAsync(CloudMessage message, TimeSpan visibilityTimeout, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
var cloudMessage = new CloudQueueMessage(message.Id, message.PopReceipt);
return(_queue.UpdateMessageAsync(cloudMessage, visibilityTimeout, MessageUpdateFields.Visibility, options, operationContext, cancellationToken));
}
public Task <CloudQueueMessage> PeekMessageAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(CloudQueue.PeekMessageAsync(options, operationContext, cancellationToken));
}
public async Task <CloudMessage> GetMessageAsync(TimeSpan?visibilityTimeout = null, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
// Get the next message from the queue
var cloudMessage = await _queue.GetMessageAsync(visibilityTimeout, options, operationContext, cancellationToken).ConfigureAwait(false);
// Convert the Azure SDK message into a Picton message
var message = await ConvertToPictonMessageAsync(cloudMessage, cancellationToken).ConfigureAwait(false);
return(message);
}
public Task SetMetadataAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(_queue.SetMetadataAsync(options, operationContext, cancellationToken));
}
/// <summary>
/// Fetch queue attributes
/// </summary>
/// <param name="queue">Cloud queue object</param>
/// <param name="options">Queue request options</param>
/// <param name="operationContext">Operation context</param>
public void FetchAttributes(CloudQueue queue, QueueRequestOptions options, OperationContext operationContext)
{
return;
}
public Task <bool> ExistsAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(_queue.ExistsAsync(options, operationContext, cancellationToken));
}
/// <summary>
/// Get queue permission
/// </summary>
/// <param name="queue">CloudQueue object</param>
/// <param name="options">Queue request options</param>
/// <param name="operationContext">Operation context</param>
/// <returns>Queue permission</returns>
public QueuePermissions GetPermissions(CloudQueue queue, QueueRequestOptions options, OperationContext operationContext)
{
throw new NotImplementedException();
}
public static void Run(
[QueueTrigger(QueueConstants.RedditSearchQueueName, Connection = QueueConstants.QueueConnectionStringName)] string processMessage,
ExecutionContext executionContext,
TraceWriter logger
)
{
logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
IConfiguration webConfiguration = new WebConfiguration(executionContext);
var log = new FunctionLog(logger, executionContext.InvocationId);
var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;
using (var kernel = new KernelFactory().GetKernel(
log,
webConfiguration,
objectLogger
))
{
var socialGist = kernel.Get <ISocialGist>();
var telemetry = kernel.Get <ITelemetryClient>();
socialGist.ResultLimitPerPage = webConfiguration.ResultLimitPerPage;
socialGist.MaximumResultsPerSearch = webConfiguration.MaximumResultsPerSearch;
SortedSet <SocialGistPostId> threadMatches = null;
using (var sgQueryTelemetry =
telemetry.StartTrackDependency("Execute Search", null, "SocialGistPostSearch"))
{
threadMatches = socialGist.MatchesForQuery(
webConfiguration.QueryTerms,
webConfiguration.QuerySortOrder,
null
).Result;
sgQueryTelemetry.IsSuccess = true;
}
logger.Info(
$"Returned [{threadMatches.Count}] posts from search terms [{webConfiguration.QueryTerms}]");
using (var queueCollectorTelemetry =
telemetry.StartTrackDependency("Enqueue Results", null, "SocialGistPostSearch"))
{
var timeDelay = webConfiguration.SearchToThreadTimeDelay;
var queue = CreateQueue();
queue.CreateIfNotExists();
QueueRequestOptions queueRequestOptions = new QueueRequestOptions()
{
MaximumExecutionTime = TimeSpan.FromMinutes(1)
};
var q = new HashSet <int>();
// Write to the queue. By default this will use will utilize however many threads the underlying scheduler provides.
// See https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.paralleloptions.maxdegreeofparallelism?view=netframework-4.7.1#System_Threading_Tasks_ParallelOptions_MaxDegreeOfParallelism
Parallel.ForEach <SocialGistPostId, CloudQueue>(
threadMatches,
CreateQueue,
(item, loopState, innerQueue) =>
{
q.Add(Thread.CurrentThread.ManagedThreadId);
var messageContent = JsonConvert.SerializeObject(item);
var message = new CloudQueueMessage(messageContent);
innerQueue.AddMessage(message, options: queueRequestOptions,
initialVisibilityDelay: timeDelay);
return(innerQueue);
},
(finalResult) => { }
);
queueCollectorTelemetry.Properties.Add("Total Number of Threads Used", q.Count.ToString());
queueCollectorTelemetry.IsSuccess = true;
}
var metric = new MetricTelemetry()
{
Name = "Unique posts returned by search",
Sum = threadMatches.Count,
Timestamp = DateTime.Now,
Properties =
{
{ "QueryTerms", webConfiguration.QueryTerms },
}
};
telemetry.TrackMetric(metric);
}
logger.Info($"{FunctionName} completed at {DateTime.Now}");
}
private static CloudQueueMessage GetMessage()
{
var options = new QueueRequestOptions();
return(_queue.GetMessageAsync(TimeSpan.FromMinutes(10), options, null).GetAwaiter().GetResult());
}
public Task UpdateMessageAsync(CloudQueueMessage message, TimeSpan visibilityTimeout, MessageUpdateFields updateFields, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(CloudQueue.UpdateMessageAsync(message, visibilityTimeout, updateFields, options, operationContext, cancellationToken));
}
public Task <IEnumerable <CloudQueueMessage> > PeekMessagesAsync(int messageCount, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(CloudQueue.PeekMessagesAsync(messageCount, options, operationContext, cancellationToken));
}
/// <summary>
/// List storage queues
/// </summary>
/// <param name="prefix">Queue name prefix</param>
/// <param name="queueListingDetails">Queue listing details</param>
/// <param name="options">Queue request options</param>
/// <param name="operationContext">Operation context</param>
/// <returns>An enumerable collection of the queues in the storage account.</returns>
public IEnumerable <CloudQueue> ListQueues(string prefix, QueueListingDetails queueListingDetails, QueueRequestOptions options, OperationContext operationContext)
{
if (string.IsNullOrEmpty(prefix))
{
return(queueList);
}
else
{
List <CloudQueue> prefixQueues = new List <CloudQueue>();
foreach (CloudQueue queue in queueList)
{
if (queue.Name.StartsWith(prefix))
{
prefixQueues.Add(queue);
}
}
return(prefixQueues);
}
}
/// <summary>
/// Create an cloud queue on azure if not exists.
/// </summary>
/// <param name="queue">Cloud queue object.</param>
/// <param name="options">Queue request options</param>
/// <param name="operationContext">Operation context</param>
/// <returns>True if the queue did not already exist and was created; otherwise false.</returns>
public bool CreateQueueIfNotExists(CloudQueue queue, QueueRequestOptions options, OperationContext operationContext)
{
return(queue.CreateIfNotExists(options, operationContext));
}
public async Task AddMessageAsync <T>(T message, TimeSpan?timeToLive = null, TimeSpan?initialVisibilityDelay = null, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
var serializer = new Serializer();
var data = (byte[])null;
using (var stream = new MemoryStream())
{
serializer.Serialize(message, stream);
data = stream.ToArray();
}
// Check if the message exceeds the size allowed by Azure Storage queues
if (data.Length > MAX_MESSAGE_CONTENT_SIZE)
{
// The message is too large. Therefore we must save the content to blob storage and
// send a smaller message indicating where the actual message was saved
// 1) Save the large message to blob storage
var blobName = $"{DateTime.UtcNow.ToString("yyyy-MM-dd")}-{RandomGenerator.GenerateString(32)}";
var blob = _blobContainer.GetBlockBlobReference(blobName);
await blob.UploadBytesAsync(data, null, cancellationToken).ConfigureAwait(false);
// 2) Send a smaller message
var largeEnvelope = new LargeMessageEnvelope
{
BlobName = blobName
};
using (var stream = new MemoryStream())
{
serializer.Serialize(largeEnvelope, stream);
data = stream.ToArray();
}
/*
* Weird issue: the C# compiler throws "CS1503 Argument 1: cannot convert from 'byte[]' to 'string'" when initializing a new message with a byte array.
* The work around is to initialize with an empty string and subsequently invoke the 'SetMessageContent' method with the byte array
* var cloudMessage = new CloudQueueMessage(data);
*/
var cloudMessage = new CloudQueueMessage(string.Empty);
cloudMessage.SetMessageContent(data);
await _queue.AddMessageAsync(cloudMessage, timeToLive, initialVisibilityDelay, options, operationContext, cancellationToken).ConfigureAwait(false);
}
else
{
// The size of this message is within the range allowed by Azure Storage queues
/*
* Weird issue: the C# compiler throws "CS1503 Argument 1: cannot convert from 'byte[]' to 'string'" when initializing a new message with a byte array.
* The work around is to initialize with an empty string and subsequently invoke the 'SetMessageContent' method with the byte array
* var cloudMessage = new CloudQueueMessage(data);
*/
var cloudMessage = new CloudQueueMessage(string.Empty);
cloudMessage.SetMessageContent(data);
await _queue.AddMessageAsync(cloudMessage, timeToLive, initialVisibilityDelay, options, operationContext, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// Delete the specified storage queue.
/// </summary>
/// <param name="queue">Cloud queue object</param>
/// <param name="options">Queue request options</param>
/// <param name="operationContext">Operation context</param>
public void DeleteQueue(CloudQueue queue, QueueRequestOptions options, OperationContext operationContext)
{
queue.Delete(options, operationContext);
}
public Task FetchAttributesAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(_queue.FetchAttributesAsync(options, operationContext, cancellationToken));
}
/// <summary>
/// Checks existence of the queue.
/// </summary>
/// <param name="queue">Cloud queue object</param>
/// <param name="requestOptions">Queue request options</param>
/// <param name="operationContext">Operation context</param>
/// <returns>True if the queue exists, otherwise false</returns>
public bool DoesQueueExist(CloudQueue queue, QueueRequestOptions requestOptions, OperationContext operationContext)
{
return(queue.Exists(requestOptions, operationContext));
}
public async Task <IEnumerable <CloudMessage> > PeekMessagesAsync(int messageCount, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
if (messageCount < 1)
{
throw new ArgumentOutOfRangeException(nameof(messageCount), "must be greather than zero");
}
if (messageCount > CloudQueueMessage.MaxNumberOfMessagesToPeek)
{
throw new ArgumentOutOfRangeException(nameof(messageCount), $"cannot be greather than {CloudQueueMessage.MaxNumberOfMessagesToPeek}");
}
// Peek at the messages from the queue
var cloudMessages = await _queue.PeekMessagesAsync(messageCount, options, operationContext, cancellationToken).ConfigureAwait(false);
// Convert the Azure SDK messages into Picton messages
return(await Task.WhenAll(from cloudMessage in cloudMessages select ConvertToPictonMessageAsync(cloudMessage, cancellationToken)).ConfigureAwait(false));
}
/// <summary>
/// Get queue permission
/// </summary>
/// <param name="options">Queue request options </param>
/// <param name="operationContext">Operation context</param>
/// <returns>QueuePermissions object</returns>
public QueuePermissions GetPermissions(CloudQueue queue, QueueRequestOptions options, OperationContext operationContext)
{
return(queue.GetPermissions(options, operationContext));
}
public Task SetPermissionsAsync(QueuePermissions permissions, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(_queue.SetPermissionsAsync(permissions, options, operationContext, cancellationToken));
}
/// <summary>
/// Get queue permission async
/// </summary>
/// <param name="queue"></param>
/// <param name="requestOptions"></param>
/// <param name="operationContext"></param>
/// <returns></returns>
public Task <QueuePermissions> GetPermissionsAsync(CloudQueue queue, QueueRequestOptions requestOptions, OperationContext operationContext)
{
return(queue.GetPermissionsAsync(requestOptions, operationContext));
}
public async Task AddMessageAsync <T>(T message, TimeSpan?timeToLive = null, TimeSpan?initialVisibilityDelay = null, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
var data = Serialize(message);
// Check if the message exceeds the size allowed by Azure Storage queues
if (data.Length > MAX_MESSAGE_CONTENT_SIZE)
{
// The message is too large. Therefore we must save the content to blob storage and
// send a smaller message indicating where the actual message was saved
// 1) Save the large message to blob storage
var blobName = $"{DateTime.UtcNow.ToString("yyyy-MM-dd")}-{RandomGenerator.GenerateString(32)}";
var blob = _blobContainer.GetBlobReference(blobName);
await blob.UploadBytesAsync(data, null, cancellationToken).ConfigureAwait(false);
// 2) Send a smaller message
var largeEnvelope = new LargeMessageEnvelope
{
BlobName = blobName
};
data = Serialize(largeEnvelope);
/*
* There is a constructor that accepts an array of bytes in NETFULL but it is not available in NETSTANDARD.
* The work around is to initialize with an empty string and subsequently invoke the 'SetMessageContent' method with the byte array
*/
var cloudMessage = new CloudQueueMessage(string.Empty);
cloudMessage.SetMessageContent(data);
await _queue.AddMessageAsync(cloudMessage, timeToLive, initialVisibilityDelay, options, operationContext, cancellationToken).ConfigureAwait(false);
}
else
{
// The size of this message is within the range allowed by Azure Storage queues
/*
* There is a constructor that accepts an array of bytes in NETFULL but it is not available in NETSTANDARD.
* The work around is to initialize with an empty string and subsequently invoke the 'SetMessageContent' method with the byte array
*/
var cloudMessage = new CloudQueueMessage(string.Empty);
cloudMessage.SetMessageContent(data);
await _queue.AddMessageAsync(cloudMessage, timeToLive, initialVisibilityDelay, options, operationContext, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// set queue permission
/// </summary>
/// <param name="queue"></param>
/// <param name="queuePermissions"></param>
/// <param name="requestOptions"></param>
/// <param name="operationContext"></param>
public void SetPermissions(CloudQueue queue, QueuePermissions queuePermissions, QueueRequestOptions requestOptions, OperationContext operationContext)
{
queue.SetPermissions(queuePermissions, requestOptions, operationContext);
}