public static async Task Run(
[QueueTrigger(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] string jobId,
ExecutionContext executionContext,
TraceWriter logger
)
{
logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
IConfiguration webConfiguration = new WebConfiguration(executionContext);
var log = new FunctionLog(logger, executionContext.InvocationId);
var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;
using (var kernel = new KernelFactory().GetKernel(
log,
webConfiguration,
objectLogger
))
{
var processor = kernel.Get <IScheduledAzureMLProcessor>();
var result = await processor.CheckAzureMLAndPostProcess(jobId);
if (!result.LastJobStatus.IsTerminalState())
{
var queue = CreateQueue();
// Job is not finished. Put it back on the queue to try again.
var message = new CloudQueueMessage(jobId);
queue.AddMessage(message, null, webConfiguration.AzureMlRetryTimeDelay);
}
}
logger.Info($"{FunctionName} completed at {DateTime.Now}");
}
public static async Task ProcessRedditPost(
[QueueTrigger(QueueConstants.RedditPostQueueName, Connection = QueueConstants.QueueConnectionStringName)] SocialGistPostId socialGistPost,
TraceWriter log,
ExecutionContext executionContext
)
{
log.Info($"{FunctionName} Execution begun at {DateTime.Now}");
var config = new WebConfiguration(executionContext);
var logger = new FunctionLog(log, executionContext.InvocationId);
var objectLogger = (config.UseObjectLogger) ? new BlobObjectLogger(config, logger) : null;
using (var kernel = new KernelFactory().GetKernel(logger, config, objectLogger))
{
var processor = kernel.Get <IThreadProcessor>();
var socialGist = kernel.Get <ISocialGist>();
socialGist.ResultLimitPerPage = config.ResultLimitPerPage;
socialGist.MaximumResultsPerSearch = config.MaximumResultsPerSearch;
await processor.Process(socialGistPost);
}
log.Info($"{FunctionName} completed at {DateTime.Now}");
}
public static async Task Run(
[TimerTrigger(ScheduledAzureMLFrequencyName, RunOnStartup = WebServiceRunConstants.RunAmlOnStartup)] TimerInfo timer, // Every half hour
[Queue(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] ICollector <string> queueCollector,
ExecutionContext executionContext,
TraceWriter logger
)
{
logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
IConfiguration webConfiguration = new WebConfiguration(executionContext);
var log = new FunctionLog(logger, executionContext.InvocationId);
var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;
using (var kernel = new KernelFactory().GetKernel(
log,
webConfiguration,
objectLogger
))
{
var processor = kernel.Get <IScheduledAzureMLProcessor>();
var result = await processor.RunAzureMLProcessing();
// If result is null then there is not any data to run through AzureML and no AML job was started.
if (result != null)
{
queueCollector.Add(result.JobId);
log.Verbose($"AzureML Web Service called; JobId=[{result.JobId}]");
}
else
{
log.Verbose("No data to run through AzureML; no AML job started.");
}
}
logger.Info($"{FunctionName} completed at {DateTime.Now}");
}
public static void Run(
[QueueTrigger(QueueConstants.RedditSearchQueueName, Connection = QueueConstants.QueueConnectionStringName)] string processMessage,
ExecutionContext executionContext,
TraceWriter logger
)
{
logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
IConfiguration webConfiguration = new WebConfiguration(executionContext);
var log = new FunctionLog(logger, executionContext.InvocationId);
var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;
using (var kernel = new KernelFactory().GetKernel(
log,
webConfiguration,
objectLogger
))
{
var socialGist = kernel.Get <ISocialGist>();
var telemetry = kernel.Get <ITelemetryClient>();
socialGist.ResultLimitPerPage = webConfiguration.ResultLimitPerPage;
socialGist.MaximumResultsPerSearch = webConfiguration.MaximumResultsPerSearch;
SortedSet <SocialGistPostId> threadMatches = null;
using (var sgQueryTelemetry =
telemetry.StartTrackDependency("Execute Search", null, "SocialGistPostSearch"))
{
threadMatches = socialGist.MatchesForQuery(
webConfiguration.QueryTerms,
webConfiguration.QuerySortOrder,
null
).Result;
sgQueryTelemetry.IsSuccess = true;
}
logger.Info(
$"Returned [{threadMatches.Count}] posts from search terms [{webConfiguration.QueryTerms}]");
using (var queueCollectorTelemetry =
telemetry.StartTrackDependency("Enqueue Results", null, "SocialGistPostSearch"))
{
var timeDelay = webConfiguration.SearchToThreadTimeDelay;
var queue = CreateQueue();
queue.CreateIfNotExists();
QueueRequestOptions queueRequestOptions = new QueueRequestOptions()
{
MaximumExecutionTime = TimeSpan.FromMinutes(1)
};
var q = new HashSet <int>();
// Write to the queue. By default this will use will utilize however many threads the underlying scheduler provides.
// See https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.paralleloptions.maxdegreeofparallelism?view=netframework-4.7.1#System_Threading_Tasks_ParallelOptions_MaxDegreeOfParallelism
Parallel.ForEach <SocialGistPostId, CloudQueue>(
threadMatches,
CreateQueue,
(item, loopState, innerQueue) =>
{
q.Add(Thread.CurrentThread.ManagedThreadId);
var messageContent = JsonConvert.SerializeObject(item);
var message = new CloudQueueMessage(messageContent);
innerQueue.AddMessage(message, options: queueRequestOptions,
initialVisibilityDelay: timeDelay);
return(innerQueue);
},
(finalResult) => { }
);
queueCollectorTelemetry.Properties.Add("Total Number of Threads Used", q.Count.ToString());
queueCollectorTelemetry.IsSuccess = true;
}
var metric = new MetricTelemetry()
{
Name = "Unique posts returned by search",
Sum = threadMatches.Count,
Timestamp = DateTime.Now,
Properties =
{
{ "QueryTerms", webConfiguration.QueryTerms },
}
};
telemetry.TrackMetric(metric);
}
logger.Info($"{FunctionName} completed at {DateTime.Now}");
}