private async Task RestartInternalAsync(OnlineTrainerSettingsInternal settings)
{
this.telemetry.TrackTrace("Online Trainer restarting", SeverityLevel.Information);
await this.StopInternalAsync();
await this.StartInternalAsync(settings);
}
private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null)
{
this.LastStartDateTimeUtc = DateTime.UtcNow;
this.perfCounters = new PerformanceCounters(settings.Metadata.ApplicationID);
// setup trainer
this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters);
if (settings.ForceFreshStart || model != null)
{
this.trainer.FreshStart(state, model);
}
else
{
await this.trainer.FindAndResumeFromState();
}
// setup factory
this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters);
// setup host
var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString);
var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath;
serviceBusConnectionStringBuilder.EntityPath = string.Empty;
this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName,
settings.JoinedEventHubConsumerGroup, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString);
// used by this.InitialOffsetProvider if no checkpointed state is found
this.eventHubStartDateTimeUtc = settings.EventHubStartDateTimeUtc;
await this.eventProcessorHost.RegisterEventProcessorFactoryAsync(
this.trainProcessorFactory,
new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider });
// don't perform too often
this.perfUpdater = new SafeTimer(
TimeSpan.FromMilliseconds(500),
this.UpdatePerformanceCounters);
var vwArgs = this.trainer.VowpalWabbit.Arguments;
this.telemetry.TrackTrace(
"OnlineTrainer started",
SeverityLevel.Information,
new Dictionary <string, string>
{
{ "CheckpointPolicy", settings.CheckpointPolicy.ToString() },
{ "VowpalWabbit", settings.Metadata.TrainArguments },
{ "ExampleTracing", settings.EnableExampleTracing.ToString() },
{ "LearningRate", vwArgs.LearningRate.ToString() },
{ "PowerT", vwArgs.PowerT.ToString() }
});
}
//private VowpalWabbitThreadedLearning vwAllReduce;
internal Learner(OnlineTrainerSettingsInternal settings, Action <VowpalWabbitJsonSerializer> delayedExampleCallback, PerformanceCounters perfCounters)
{
this.telemetry = new TelemetryClient();
this.settings = settings;
this.delayedExampleCallback = delayedExampleCallback;
this.perfCounters = perfCounters;
this.trackbackList = new List <string>();
this.blobClient = CloudStorageAccount.Parse(settings.StorageConnectionString).CreateCloudBlobClient();
}
/// <summary>
/// Restarts the trainer.
/// </summary>
public async Task Restart(OnlineTrainerSettingsInternal settings)
{
await this.SafeExecute(async() => await this.RestartInternalAsync(settings));
}