internal void FreshStart(OnlineTrainerState state = null)
{
if (state == null)
state = new OnlineTrainerState();
this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information);
// start from scratch
this.state = state;
this.InitializeVowpalWabbit(new VowpalWabbitSettings(this.settings.Metadata.TrainArguments));
}
internal void FreshStart(OnlineTrainerState state = null)
{
if (state == null)
{
state = new OnlineTrainerState();
}
this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information);
// start from scratch
this.state = state;
this.InitializeVowpalWabbit(new VowpalWabbitSettings(this.settings.Metadata.TrainArguments));
}
internal void FreshStart(OnlineTrainerState state = null, byte[] model = null)
{
if (state == null)
state = new OnlineTrainerState();
this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information);
// start from scratch
this.state = state;
var settings = model == null ?
new VowpalWabbitSettings(this.settings.Metadata.TrainArguments) :
new VowpalWabbitSettings(string.Empty) { ModelStream = new MemoryStream(model) };
this.InitializeVowpalWabbit(settings);
}
private async Task ResetInternalAsync(OnlineTrainerState state = null, byte[] model = null)
{
if (this.trainer == null)
{
this.telemetry.TrackTrace("Online Trainer resetting skipped as trainer hasn't started yet.", SeverityLevel.Information);
return;
}
var msg = "Online Trainer resetting";
bool updateClientModel = false;
if (state != null)
{
msg += "; state supplied";
updateClientModel = true;
}
if (model != null)
{
msg += $"; model of size {model.Length} supplied.";
updateClientModel = true;
}
this.telemetry.TrackTrace(msg, SeverityLevel.Information);
var settings = this.trainer.Settings;
await this.StopInternalAsync();
settings.ForceFreshStart = true;
settings.CheckpointPolicy.Reset();
await this.StartInternalAsync(settings, state, model);
// make sure we store this fresh model, in case we die we don't loose the reset
await this.trainProcessorFactory.LearnBlock.SendAsync(new CheckpointTriggerEvent { UpdateClientModel = updateClientModel });
if (!updateClientModel)
{
// delete the currently deployed model, so the clients don't use the hold one
var latestModel = await this.trainer.GetLatestModelBlob();
this.telemetry.TrackTrace($"Resetting client visible model: {latestModel.Uri}", SeverityLevel.Information);
await latestModel.UploadFromByteArrayAsync(new byte[0], 0, 0);
}
}
internal void FreshStart(OnlineTrainerState state = null, byte[] model = null)
{
if (state == null)
{
state = new OnlineTrainerState();
}
this.telemetry.TrackTrace("Fresh Start", SeverityLevel.Information);
// start from scratch
this.state = state;
var settings = model == null ?
new VowpalWabbitSettings(this.settings.Metadata.TrainArguments) :
new VowpalWabbitSettings(string.Empty)
{
ModelStream = new MemoryStream(model)
};
this.InitializeVowpalWabbit(settings);
}
private async Task ResetInternalAsync(OnlineTrainerState state = null)
{
this.telemetry.TrackTrace("Online Trainer resetting", SeverityLevel.Information);
var settings = this.trainer.Settings;
await this.StopInternalAsync();
settings.ForceFreshStart = true;
settings.CheckpointPolicy.Reset();
await this.StartInternalAsync(settings, state);
// make sure we store this fresh model, in case we die we don't loose the reset
await this.trainProcessorFactory.LearnBlock.SendAsync(new CheckpointTriggerEvent());
// delete the currently deployed model, so the clients don't use the hold one
var latestModel = await this.trainer.GetLatestModelBlob();
this.telemetry.TrackTrace($"Resetting client visible model: {latestModel.Uri}", SeverityLevel.Information);
await latestModel.UploadFromByteArrayAsync(new byte[0], 0, 0);
}
public async Task ResetModelAsync(OnlineTrainerState state = null, byte[] model = null)
{
await this.SafeExecute(async() => await this.ResetInternalAsync(state, model));
}
private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null)
{
this.LastStartDateTimeUtc = DateTime.UtcNow;
this.perfCounters = new PerformanceCounters(settings.Metadata.ApplicationID);
// setup trainer
this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters);
if (settings.ForceFreshStart || model != null)
{
this.trainer.FreshStart(state, model);
}
else
{
await this.trainer.FindAndResumeFromState();
}
// setup factory
this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters);
// setup host
var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString);
var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath;
serviceBusConnectionStringBuilder.EntityPath = string.Empty;
this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName,
EventHubConsumerGroup.DefaultGroupName, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString);
await this.eventProcessorHost.RegisterEventProcessorFactoryAsync(
this.trainProcessorFactory,
new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider });
// don't perform too often
this.perfUpdater = new SafeTimer(
TimeSpan.FromMilliseconds(500),
this.UpdatePerformanceCounters);
this.telemetry.TrackTrace(
"OnlineTrainer started",
SeverityLevel.Information,
new Dictionary <string, string>
{
{ "CheckpointPolicy", settings.CheckpointPolicy.ToString() },
{ "VowpalWabbit", settings.Metadata.TrainArguments },
{ "ExampleTracing", settings.EnableExampleTracing.ToString() }
});
}
private async Task ResetInternalAsync(OnlineTrainerState state = null, byte[] model = null)
{
if (this.trainer == null)
{
this.telemetry.TrackTrace("Online Trainer resetting skipped as trainer hasn't started yet.", SeverityLevel.Information);
return;
}
var msg = "Online Trainer resetting";
bool updateClientModel = false;
if (state != null)
{
msg += "; state supplied";
updateClientModel = true;
}
if (model != null)
{
msg += $"; model of size {model.Length} supplied.";
updateClientModel = true;
}
this.telemetry.TrackTrace(msg, SeverityLevel.Information);
var settings = this.trainer.Settings;
await this.StopInternalAsync();
settings.ForceFreshStart = true;
settings.CheckpointPolicy.Reset();
await this.StartInternalAsync(settings, state, model);
// make sure we store this fresh model, in case we die we don't loose the reset
await this.trainProcessorFactory.LearnBlock.SendAsync(new CheckpointTriggerEvent { UpdateClientModel = updateClientModel });
if (!updateClientModel)
{
// delete the currently deployed model, so the clients don't use the hold one
var latestModel = await this.trainer.GetLatestModelBlob();
this.telemetry.TrackTrace($"Resetting client visible model: {latestModel.Uri}", SeverityLevel.Information);
await latestModel.UploadFromByteArrayAsync(new byte[0], 0, 0);
}
}
public async Task ResetModelAsync(OnlineTrainerState state = null, byte[] model = null)
{
await this.SafeExecute(async () => await this.ResetInternalAsync(state, model));
}
private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null)
{
this.LastStartDateTimeUtc = DateTime.UtcNow;
this.perfCounters = new PerformanceCounters(settings.Metadata.ApplicationID);
// setup trainer
this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters);
if (settings.ForceFreshStart || model != null)
this.trainer.FreshStart(state, model);
else
await this.trainer.FindAndResumeFromState();
// setup factory
this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters);
// setup host
var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString);
var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath;
serviceBusConnectionStringBuilder.EntityPath = string.Empty;
this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName,
EventHubConsumerGroup.DefaultGroupName, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString);
await this.eventProcessorHost.RegisterEventProcessorFactoryAsync(
this.trainProcessorFactory,
new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider });
// don't perform too often
this.perfUpdater = new SafeTimer(
TimeSpan.FromMilliseconds(500),
this.UpdatePerformanceCounters);
this.telemetry.TrackTrace(
"OnlineTrainer started",
SeverityLevel.Information,
new Dictionary<string, string>
{
{ "CheckpointPolicy", settings.CheckpointPolicy.ToString() },
{ "VowpalWabbit", settings.Metadata.TrainArguments },
{ "ExampleTracing", settings.EnableExampleTracing.ToString() }
});
}