private async Task StartInternalAsync(OnlineTrainerSettingsInternal settings, OnlineTrainerState state = null, byte[] model = null)
{
this.LastStartDateTimeUtc = DateTime.UtcNow;
this.perfCounters = new PerformanceCounters(settings.Metadata.ApplicationID);
// setup trainer
this.trainer = new Learner(settings, this.DelayedExampleCallback, this.perfCounters);
if (settings.ForceFreshStart || model != null)
{
this.trainer.FreshStart(state, model);
}
else
{
await this.trainer.FindAndResumeFromState();
}
// setup factory
this.trainProcessorFactory = new TrainEventProcessorFactory(settings, this.trainer, this.perfCounters);
// setup host
var serviceBusConnectionStringBuilder = new ServiceBusConnectionStringBuilder(settings.JoinedEventHubConnectionString);
var joinedEventhubName = serviceBusConnectionStringBuilder.EntityPath;
serviceBusConnectionStringBuilder.EntityPath = string.Empty;
this.eventProcessorHost = new EventProcessorHost(settings.Metadata.ApplicationID, joinedEventhubName,
settings.JoinedEventHubConsumerGroup, serviceBusConnectionStringBuilder.ToString(), settings.StorageConnectionString);
// used by this.InitialOffsetProvider if no checkpointed state is found
this.eventHubStartDateTimeUtc = settings.EventHubStartDateTimeUtc;
await this.eventProcessorHost.RegisterEventProcessorFactoryAsync(
this.trainProcessorFactory,
new EventProcessorOptions { InitialOffsetProvider = this.InitialOffsetProvider });
// don't perform too often
this.perfUpdater = new SafeTimer(
TimeSpan.FromMilliseconds(500),
this.UpdatePerformanceCounters);
var vwArgs = this.trainer.VowpalWabbit.Arguments;
this.telemetry.TrackTrace(
"OnlineTrainer started",
SeverityLevel.Information,
new Dictionary <string, string>
{
{ "CheckpointPolicy", settings.CheckpointPolicy.ToString() },
{ "VowpalWabbit", settings.Metadata.TrainArguments },
{ "ExampleTracing", settings.EnableExampleTracing.ToString() },
{ "LearningRate", vwArgs.LearningRate.ToString() },
{ "PowerT", vwArgs.PowerT.ToString() }
});
}
//private VowpalWabbitThreadedLearning vwAllReduce;
internal Learner(OnlineTrainerSettingsInternal settings, Action <VowpalWabbitJsonSerializer> delayedExampleCallback, PerformanceCounters perfCounters)
{
this.telemetry = new TelemetryClient();
this.settings = settings;
this.delayedExampleCallback = delayedExampleCallback;
this.perfCounters = perfCounters;
this.trackbackList = new List <string>();
this.blobClient = CloudStorageAccount.Parse(settings.StorageConnectionString).CreateCloudBlobClient();
}
private async Task StopInternalAsync()
{
this.telemetry.TrackTrace("OnlineTrainer stopping", SeverityLevel.Verbose);
if (this.perfUpdater != null)
{
this.perfUpdater.Stop(TimeSpan.FromMinutes(1));
this.perfUpdater = null;
}
if (this.eventProcessorHost != null)
{
try
{
await this.eventProcessorHost.UnregisterEventProcessorAsync();
}
catch (Exception ex)
{
this.telemetry.TrackException(ex);
}
this.eventProcessorHost = null;
}
if (this.trainProcessorFactory != null)
{
// flushes the pipeline
this.trainProcessorFactory.Dispose();
this.trainProcessorFactory = null;
}
if (this.trainer != null)
{
this.trainer.Dispose();
this.trainer = null;
}
if (this.perfCounters != null)
{
this.perfCounters.Dispose();
this.perfCounters = null;
}
this.telemetry.TrackTrace("OnlineTrainer stopped", SeverityLevel.Verbose);
}