public static async Task Run( [QueueTrigger(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] string jobId, ExecutionContext executionContext, TraceWriter logger ) { logger.Info($"{FunctionName} Execution begun at {DateTime.Now}"); IConfiguration webConfiguration = new WebConfiguration(executionContext); var log = new FunctionLog(logger, executionContext.InvocationId); var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null; using (var kernel = new KernelFactory().GetKernel( log, webConfiguration, objectLogger )) { var processor = kernel.Get <IScheduledAzureMLProcessor>(); var result = await processor.CheckAzureMLAndPostProcess(jobId); if (!result.LastJobStatus.IsTerminalState()) { var queue = CreateQueue(); // Job is not finished. Put it back on the queue to try again. var message = new CloudQueueMessage(jobId); queue.AddMessage(message, null, webConfiguration.AzureMlRetryTimeDelay); } } logger.Info($"{FunctionName} completed at {DateTime.Now}"); }
public static async Task ProcessRedditPost( [QueueTrigger(QueueConstants.RedditPostQueueName, Connection = QueueConstants.QueueConnectionStringName)] SocialGistPostId socialGistPost, TraceWriter log, ExecutionContext executionContext ) { log.Info($"{FunctionName} Execution begun at {DateTime.Now}"); var config = new WebConfiguration(executionContext); var logger = new FunctionLog(log, executionContext.InvocationId); var objectLogger = (config.UseObjectLogger) ? new BlobObjectLogger(config, logger) : null; using (var kernel = new KernelFactory().GetKernel(logger, config, objectLogger)) { var processor = kernel.Get <IThreadProcessor>(); var socialGist = kernel.Get <ISocialGist>(); socialGist.ResultLimitPerPage = config.ResultLimitPerPage; socialGist.MaximumResultsPerSearch = config.MaximumResultsPerSearch; await processor.Process(socialGistPost); } log.Info($"{FunctionName} completed at {DateTime.Now}"); }
public static async Task Run( [TimerTrigger(ScheduledAzureMLFrequencyName, RunOnStartup = WebServiceRunConstants.RunAmlOnStartup)] TimerInfo timer, // Every half hour [Queue(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] ICollector <string> queueCollector, ExecutionContext executionContext, TraceWriter logger ) { logger.Info($"{FunctionName} Execution begun at {DateTime.Now}"); IConfiguration webConfiguration = new WebConfiguration(executionContext); var log = new FunctionLog(logger, executionContext.InvocationId); var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null; using (var kernel = new KernelFactory().GetKernel( log, webConfiguration, objectLogger )) { var processor = kernel.Get <IScheduledAzureMLProcessor>(); var result = await processor.RunAzureMLProcessing(); // If result is null then there is not any data to run through AzureML and no AML job was started. if (result != null) { queueCollector.Add(result.JobId); log.Verbose($"AzureML Web Service called; JobId=[{result.JobId}]"); } else { log.Verbose("No data to run through AzureML; no AML job started."); } } logger.Info($"{FunctionName} completed at {DateTime.Now}"); }
public static void Run( [QueueTrigger(QueueConstants.RedditSearchQueueName, Connection = QueueConstants.QueueConnectionStringName)] string processMessage, ExecutionContext executionContext, TraceWriter logger ) { logger.Info($"{FunctionName} Execution begun at {DateTime.Now}"); IConfiguration webConfiguration = new WebConfiguration(executionContext); var log = new FunctionLog(logger, executionContext.InvocationId); var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null; using (var kernel = new KernelFactory().GetKernel( log, webConfiguration, objectLogger )) { var socialGist = kernel.Get <ISocialGist>(); var telemetry = kernel.Get <ITelemetryClient>(); socialGist.ResultLimitPerPage = webConfiguration.ResultLimitPerPage; socialGist.MaximumResultsPerSearch = webConfiguration.MaximumResultsPerSearch; SortedSet <SocialGistPostId> threadMatches = null; using (var sgQueryTelemetry = telemetry.StartTrackDependency("Execute Search", null, "SocialGistPostSearch")) { threadMatches = socialGist.MatchesForQuery( webConfiguration.QueryTerms, webConfiguration.QuerySortOrder, null ).Result; sgQueryTelemetry.IsSuccess = true; } logger.Info( $"Returned [{threadMatches.Count}] posts from search terms [{webConfiguration.QueryTerms}]"); using (var queueCollectorTelemetry = telemetry.StartTrackDependency("Enqueue Results", null, "SocialGistPostSearch")) { var timeDelay = webConfiguration.SearchToThreadTimeDelay; var queue = CreateQueue(); queue.CreateIfNotExists(); QueueRequestOptions queueRequestOptions = new QueueRequestOptions() { MaximumExecutionTime = TimeSpan.FromMinutes(1) }; var q = new HashSet <int>(); // Write to the queue. By default this will use will utilize however many threads the underlying scheduler provides. // See https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.paralleloptions.maxdegreeofparallelism?view=netframework-4.7.1#System_Threading_Tasks_ParallelOptions_MaxDegreeOfParallelism Parallel.ForEach <SocialGistPostId, CloudQueue>( threadMatches, CreateQueue, (item, loopState, innerQueue) => { q.Add(Thread.CurrentThread.ManagedThreadId); var messageContent = JsonConvert.SerializeObject(item); var message = new CloudQueueMessage(messageContent); innerQueue.AddMessage(message, options: queueRequestOptions, initialVisibilityDelay: timeDelay); return(innerQueue); }, (finalResult) => { } ); queueCollectorTelemetry.Properties.Add("Total Number of Threads Used", q.Count.ToString()); queueCollectorTelemetry.IsSuccess = true; } var metric = new MetricTelemetry() { Name = "Unique posts returned by search", Sum = threadMatches.Count, Timestamp = DateTime.Now, Properties = { { "QueryTerms", webConfiguration.QueryTerms }, } }; telemetry.TrackMetric(metric); } logger.Info($"{FunctionName} completed at {DateTime.Now}"); }