internal void FreshStart(OnlineTrainerState state = null) { if (state == null) state = new OnlineTrainerState(); this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information); // start from scratch this.state = state; this.InitializeVowpalWabbit(new VowpalWabbitSettings(this.settings.Metadata.TrainArguments)); }
internal void FreshStart(OnlineTrainerState state = null) { if (state == null) { state = new OnlineTrainerState(); } this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information); // start from scratch this.state = state; this.InitializeVowpalWabbit(new VowpalWabbitSettings(this.settings.Metadata.TrainArguments)); }
internal void FreshStart(OnlineTrainerState state = null, byte[] model = null) { if (state == null) state = new OnlineTrainerState(); this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information); // start from scratch this.state = state; var settings = model == null ? new VowpalWabbitSettings(this.settings.Metadata.TrainArguments) : new VowpalWabbitSettings(string.Empty) { ModelStream = new MemoryStream(model) }; this.InitializeVowpalWabbit(settings); }
private async Task ResetInternalAsync(OnlineTrainerState state = null, byte[] model = null) { if (this.trainer == null) { this.telemetry.TrackTrace("Online Trainer resetting skipped as trainer hasn't started yet.", SeverityLevel.Information); return; } var msg = "Online Trainer resetting"; bool updateClientModel = false; if (state != null) { msg += "; state supplied"; updateClientModel = true; } if (model != null) { msg += $"; model of size {model.Length} supplied."; updateClientModel = true; } this.telemetry.TrackTrace(msg, SeverityLevel.Information); var settings = this.trainer.Settings; await this.StopInternalAsync(); settings.ForceFreshStart = true; settings.CheckpointPolicy.Reset(); await this.StartInternalAsync(settings, state, model); // make sure we store this fresh model, in case we die we don't loose the reset await this.trainProcessorFactory.LearnBlock.SendAsync(new CheckpointTriggerEvent { UpdateClientModel = updateClientModel }); if (!updateClientModel) { // delete the currently deployed model, so the clients don't use the hold one var latestModel = await this.trainer.GetLatestModelBlob(); this.telemetry.TrackTrace($"Resetting client visible model: {latestModel.Uri}", SeverityLevel.Information); await latestModel.UploadFromByteArrayAsync(new byte[0], 0, 0); } }
internal void FreshStart(OnlineTrainerState state = null, byte[] model = null) { if (state == null) { state = new OnlineTrainerState(); } this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information); // start from scratch this.state = state; var settings = model == null ? new VowpalWabbitSettings(this.settings.Metadata.TrainArguments) : new VowpalWabbitSettings(string.Empty) { ModelStream = new MemoryStream(model) }; this.InitializeVowpalWabbit(settings); }
private async Task ResetInternalAsync(OnlineTrainerState state = null) { this.telemetry.TrackTrace("Online Trainer resetting", SeverityLevel.Information); var settings = this.trainer.Settings; await this.StopInternalAsync(); settings.ForceFreshStart = true; settings.CheckpointPolicy.Reset(); await this.StartInternalAsync(settings, state); // make sure we store this fresh model, in case we die we don't loose the reset await this.trainProcessorFactory.LearnBlock.SendAsync(new CheckpointTriggerEvent()); // delete the currently deployed model, so the clients don't use the hold one var latestModel = await this.trainer.GetLatestModelBlob(); this.telemetry.TrackTrace($"Resetting client visible model: {latestModel.Uri}", SeverityLevel.Information); await latestModel.UploadFromByteArrayAsync(new byte[0], 0, 0); }
public async Task ResetModelAsync(OnlineTrainerState state = null, byte[] model = null) { await this.SafeExecute(async() => await this.ResetInternalAsync(state, model)); }
private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null) { this.LastStartDateTimeUtc = DateTime.UtcNow; this.perfCounters = new PerformanceCounters(settings.Metadata.ApplicationID); // setup trainer this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters); if (settings.ForceFreshStart || model != null) { this.trainer.FreshStart(state, model); } else { await this.trainer.FindAndResumeFromState(); } // setup factory this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters); // setup host var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString); var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath; serviceBusConnectionStringBuilder.EntityPath = string.Empty; this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName, EventHubConsumerGroup.DefaultGroupName, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString); await this.eventProcessorHost.RegisterEventProcessorFactoryAsync( this.trainProcessorFactory, new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider }); // don't perform too often this.perfUpdater = new SafeTimer( TimeSpan.FromMilliseconds(500), this.UpdatePerformanceCounters); this.telemetry.TrackTrace( "OnlineTrainer started", SeverityLevel.Information, new Dictionary <string, string> { { "CheckpointPolicy", settings.CheckpointPolicy.ToString() }, { "VowpalWabbit", settings.Metadata.TrainArguments }, { "ExampleTracing", settings.EnableExampleTracing.ToString() } }); }
public async Task ResetModelAsync(OnlineTrainerState state = null, byte[] model = null) { await this.SafeExecute(async () => await this.ResetInternalAsync(state, model)); }
private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null) { this.LastStartDateTimeUtc = DateTime.UtcNow; this.perfCounters = new PerformanceCounters(settings.Metadata.ApplicationID); // setup trainer this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters); if (settings.ForceFreshStart || model != null) this.trainer.FreshStart(state, model); else await this.trainer.FindAndResumeFromState(); // setup factory this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters); // setup host var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString); var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath; serviceBusConnectionStringBuilder.EntityPath = string.Empty; this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName, EventHubConsumerGroup.DefaultGroupName, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString); await this.eventProcessorHost.RegisterEventProcessorFactoryAsync( this.trainProcessorFactory, new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider }); // don't perform too often this.perfUpdater = new SafeTimer( TimeSpan.FromMilliseconds(500), this.UpdatePerformanceCounters); this.telemetry.TrackTrace( "OnlineTrainer started", SeverityLevel.Information, new Dictionary<string, string> { { "CheckpointPolicy", settings.CheckpointPolicy.ToString() }, { "VowpalWabbit", settings.Metadata.TrainArguments }, { "ExampleTracing", settings.EnableExampleTracing.ToString() } }); }