private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null) { this.LastStartDateTimeUtc = DateTime.UtcNow; this.perfCounters = new PerformanceCounters(settings.Metadata.ApplicationID); // setup trainer this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters); if (settings.ForceFreshStart || model != null) { this.trainer.FreshStart(state, model); } else { await this.trainer.FindAndResumeFromState(); } // setup factory this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters); // setup host var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString); var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath; serviceBusConnectionStringBuilder.EntityPath = string.Empty; this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName, settings.JoinedEventHubConsumerGroup, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString); // used by this.InitialOffsetProvider if no checkpointed state is found this.eventHubStartDateTimeUtc = settings.EventHubStartDateTimeUtc; await this.eventProcessorHost.RegisterEventProcessorFactoryAsync( this.trainProcessorFactory, new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider }); // don't perform too often this.perfUpdater = new SafeTimer( TimeSpan.FromMilliseconds(500), this.UpdatePerformanceCounters); var vwArgs = this.trainer.VowpalWabbit.Arguments; this.telemetry.TrackTrace( "OnlineTrainer started", SeverityLevel.Information, new Dictionary <string, string> { { "CheckpointPolicy", settings.CheckpointPolicy.ToString() }, { "VowpalWabbit", settings.Metadata.TrainArguments }, { "ExampleTracing", settings.EnableExampleTracing.ToString() }, { "LearningRate", vwArgs.LearningRate.ToString() }, { "PowerT", vwArgs.PowerT.ToString() } }); }
//private VowpalWabbitThreadedLearning vwAllReduce; internal Learner(OnlineTrainerSettingsInternal settings, Action <VowpalWabbitJsonSerializer> delayedExampleCallback, PerformanceCounters perfCounters) { this.telemetry = new TelemetryClient(); this.settings = settings; this.delayedExampleCallback = delayedExampleCallback; this.perfCounters = perfCounters; this.trackbackList = new List <string>(); this.blobClient = CloudStorageAccount.Parse(settings.StorageConnectionString).CreateCloudBlobClient(); }
private async Task StopInternalAsync() { this.telemetry.TrackTrace("OnlineTrainer stopping", SeverityLevel.Verbose); if (this.perfUpdater != null) { this.perfUpdater.Stop(TimeSpan.FromMinutes(1)); this.perfUpdater = null; } if (this.eventProcessorHost != null) { try { await this.eventProcessorHost.UnregisterEventProcessorAsync(); } catch (Exception ex) { this.telemetry.TrackException(ex); } this.eventProcessorHost = null; } if (this.trainProcessorFactory != null) { // flushes the pipeline this.trainProcessorFactory.Dispose(); this.trainProcessorFactory = null; } if (this.trainer != null) { this.trainer.Dispose(); this.trainer = null; } if (this.perfCounters != null) { this.perfCounters.Dispose(); this.perfCounters = null; } this.telemetry.TrackTrace("OnlineTrainer stopped", SeverityLevel.Verbose); }