public static async Task Run(
            [QueueTrigger(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] string jobId,
            ExecutionContext executionContext,
            TraceWriter logger
            )
        {
            logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
            IConfiguration webConfiguration = new WebConfiguration(executionContext);
            var            log = new FunctionLog(logger, executionContext.InvocationId);

            var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;

            using (var kernel = new KernelFactory().GetKernel(
                       log,
                       webConfiguration,
                       objectLogger
                       ))
            {
                var processor = kernel.Get <IScheduledAzureMLProcessor>();
                var result    = await processor.CheckAzureMLAndPostProcess(jobId);

                if (!result.LastJobStatus.IsTerminalState())
                {
                    var queue = CreateQueue();

                    // Job is not finished.  Put it back on the queue to try again.
                    var message = new CloudQueueMessage(jobId);
                    queue.AddMessage(message, null, webConfiguration.AzureMlRetryTimeDelay);
                }
            }

            logger.Info($"{FunctionName} completed at {DateTime.Now}");
        }
        public static async Task ProcessRedditPost(
            [QueueTrigger(QueueConstants.RedditPostQueueName, Connection = QueueConstants.QueueConnectionStringName)] SocialGistPostId socialGistPost,
            TraceWriter log,
            ExecutionContext executionContext
            )
        {
            log.Info($"{FunctionName} Execution begun at {DateTime.Now}");

            var config       = new WebConfiguration(executionContext);
            var logger       = new FunctionLog(log, executionContext.InvocationId);
            var objectLogger = (config.UseObjectLogger) ? new BlobObjectLogger(config, logger) : null;

            using (var kernel = new KernelFactory().GetKernel(logger, config, objectLogger))
            {
                var processor  = kernel.Get <IThreadProcessor>();
                var socialGist = kernel.Get <ISocialGist>();
                socialGist.ResultLimitPerPage      = config.ResultLimitPerPage;
                socialGist.MaximumResultsPerSearch = config.MaximumResultsPerSearch;

                await processor.Process(socialGistPost);
            }

            log.Info($"{FunctionName} completed at {DateTime.Now}");
        }
        public static async Task Run(
            [TimerTrigger(ScheduledAzureMLFrequencyName, RunOnStartup = WebServiceRunConstants.RunAmlOnStartup)] TimerInfo timer, // Every half hour
            [Queue(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] ICollector <string> queueCollector,
            ExecutionContext executionContext,
            TraceWriter logger
            )
        {
            logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
            IConfiguration webConfiguration = new WebConfiguration(executionContext);
            var            log = new FunctionLog(logger, executionContext.InvocationId);

            var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;

            using (var kernel = new KernelFactory().GetKernel(
                       log,
                       webConfiguration,
                       objectLogger
                       ))
            {
                var processor = kernel.Get <IScheduledAzureMLProcessor>();
                var result    = await processor.RunAzureMLProcessing();

                // If result is null then there is not any data to run through AzureML and no AML job was started.
                if (result != null)
                {
                    queueCollector.Add(result.JobId);
                    log.Verbose($"AzureML Web Service called; JobId=[{result.JobId}]");
                }
                else
                {
                    log.Verbose("No data to run through AzureML; no AML job started.");
                }
            }

            logger.Info($"{FunctionName} completed at {DateTime.Now}");
        }
Exemple #4
0
        public static void Run(
            [QueueTrigger(QueueConstants.RedditSearchQueueName, Connection = QueueConstants.QueueConnectionStringName)] string processMessage,
            ExecutionContext executionContext,
            TraceWriter logger
            )
        {
            logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
            IConfiguration webConfiguration = new WebConfiguration(executionContext);
            var            log = new FunctionLog(logger, executionContext.InvocationId);

            var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;

            using (var kernel = new KernelFactory().GetKernel(
                       log,
                       webConfiguration,
                       objectLogger
                       ))
            {
                var socialGist = kernel.Get <ISocialGist>();
                var telemetry  = kernel.Get <ITelemetryClient>();
                socialGist.ResultLimitPerPage      = webConfiguration.ResultLimitPerPage;
                socialGist.MaximumResultsPerSearch = webConfiguration.MaximumResultsPerSearch;

                SortedSet <SocialGistPostId> threadMatches = null;
                using (var sgQueryTelemetry =
                           telemetry.StartTrackDependency("Execute Search", null, "SocialGistPostSearch"))
                {
                    threadMatches = socialGist.MatchesForQuery(
                        webConfiguration.QueryTerms,
                        webConfiguration.QuerySortOrder,
                        null
                        ).Result;

                    sgQueryTelemetry.IsSuccess = true;
                }

                logger.Info(
                    $"Returned [{threadMatches.Count}] posts from search terms [{webConfiguration.QueryTerms}]");

                using (var queueCollectorTelemetry =
                           telemetry.StartTrackDependency("Enqueue Results", null, "SocialGistPostSearch"))
                {
                    var timeDelay = webConfiguration.SearchToThreadTimeDelay;
                    var queue     = CreateQueue();

                    queue.CreateIfNotExists();

                    QueueRequestOptions queueRequestOptions = new QueueRequestOptions()
                    {
                        MaximumExecutionTime = TimeSpan.FromMinutes(1)
                    };

                    var q = new HashSet <int>();

                    // Write to the queue.  By default this will use will utilize however many threads the underlying scheduler provides.
                    // See https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.paralleloptions.maxdegreeofparallelism?view=netframework-4.7.1#System_Threading_Tasks_ParallelOptions_MaxDegreeOfParallelism
                    Parallel.ForEach <SocialGistPostId, CloudQueue>(
                        threadMatches,
                        CreateQueue,
                        (item, loopState, innerQueue) =>
                    {
                        q.Add(Thread.CurrentThread.ManagedThreadId);
                        var messageContent = JsonConvert.SerializeObject(item);
                        var message        = new CloudQueueMessage(messageContent);
                        innerQueue.AddMessage(message, options: queueRequestOptions,
                                              initialVisibilityDelay: timeDelay);
                        return(innerQueue);
                    },
                        (finalResult) => { }
                        );

                    queueCollectorTelemetry.Properties.Add("Total Number of Threads Used", q.Count.ToString());
                    queueCollectorTelemetry.IsSuccess = true;
                }

                var metric = new MetricTelemetry()
                {
                    Name       = "Unique posts returned by search",
                    Sum        = threadMatches.Count,
                    Timestamp  = DateTime.Now,
                    Properties =
                    {
                        { "QueryTerms", webConfiguration.QueryTerms },
                    }
                };
                telemetry.TrackMetric(metric);
            }

            logger.Info($"{FunctionName} completed at {DateTime.Now}");
        }