Exemple #1
0
        private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null)
        {
            this.LastStartDateTimeUtc = DateTime.UtcNow;
            this.perfCounters         = new PerformanceCounters(settings.Metadata.ApplicationID);

            // setup trainer
            this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters);

            if (settings.ForceFreshStart || model != null)
            {
                this.trainer.FreshStart(state, model);
            }
            else
            {
                await this.trainer.FindAndResumeFromState();
            }

            // setup factory
            this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters);

            // setup host
            var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString);
            var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath;

            serviceBusConnectionStringBuilder.EntityPath = string.Empty;

            this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName,
                                                             settings.JoinedEventHubConsumerGroup, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString);

            // used by this.InitialOffsetProvider if no checkpointed state is found
            this.eventHubStartDateTimeUtc = settings.EventHubStartDateTimeUtc;

            await this.eventProcessorHost.RegisterEventProcessorFactoryAsync(
                this.trainProcessorFactory,
                new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider });

            // don't perform too often
            this.perfUpdater = new SafeTimer(
                TimeSpan.FromMilliseconds(500),
                this.UpdatePerformanceCounters);

            var vwArgs = this.trainer.VowpalWabbit.Arguments;

            this.telemetry.TrackTrace(
                "OnlineTrainer started",
                SeverityLevel.Information,
                new Dictionary <string, string>
            {
                { "CheckpointPolicy", settings.CheckpointPolicy.ToString() },
                { "VowpalWabbit", settings.Metadata.TrainArguments },
                { "ExampleTracing", settings.EnableExampleTracing.ToString() },
                { "LearningRate", vwArgs.LearningRate.ToString() },
                { "PowerT", vwArgs.PowerT.ToString() }
            });
        }
Exemple #2
0
        //private VowpalWabbitThreadedLearning vwAllReduce;

        internal Learner(OnlineTrainerSettingsInternal settings, Action <VowpalWabbitJsonSerializer> delayedExampleCallback, PerformanceCounters perfCounters)
        {
            this.telemetry = new TelemetryClient();

            this.settings = settings;
            this.delayedExampleCallback = delayedExampleCallback;
            this.perfCounters           = perfCounters;

            this.trackbackList = new List <string>();
            this.blobClient    = CloudStorageAccount.Parse(settings.StorageConnectionString).CreateCloudBlobClient();
        }
Exemple #3
0
        private async Task StopInternalAsync()
        {
            this.telemetry.TrackTrace("OnlineTrainer stopping", SeverityLevel.Verbose);

            if (this.perfUpdater != null)
            {
                this.perfUpdater.Stop(TimeSpan.FromMinutes(1));
                this.perfUpdater = null;
            }

            if (this.eventProcessorHost != null)
            {
                try
                {
                    await this.eventProcessorHost.UnregisterEventProcessorAsync();
                }
                catch (Exception ex)
                {
                    this.telemetry.TrackException(ex);
                }

                this.eventProcessorHost = null;
            }

            if (this.trainProcessorFactory != null)
            {
                // flushes the pipeline
                this.trainProcessorFactory.Dispose();
                this.trainProcessorFactory = null;
            }

            if (this.trainer != null)
            {
                this.trainer.Dispose();
                this.trainer = null;
            }

            if (this.perfCounters != null)
            {
                this.perfCounters.Dispose();
                this.perfCounters = null;
            }

            this.telemetry.TrackTrace("OnlineTrainer stopped", SeverityLevel.Verbose);
        }