public override bool OnStart()
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 128;
// For information on handling configuration changes
// see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357.
bool result = base.OnStart();
TelemetryConfiguration.Active.InstrumentationKey = CloudConfigurationManager.GetSetting("APPINSIGHTS_INSTRUMENTATIONKEY");
// TODO: disable
// TelemetryConfiguration.Active.TelemetryChannel.DeveloperMode = true;
this.telemetry = new TelemetryClient();
this.telemetry.TrackTrace("WorkerRole starting", SeverityLevel.Information);
try
{
this.trainProcesserHost = new LearnEventProcessorHost();
this.settingsWatcher = new OnlineTrainerSettingsWatcher(this.trainProcesserHost);
this.StartRESTAdminEndpoint();
}
catch (Exception e)
{
this.telemetry.TrackException(e);
// still start to give AppInsights a chance to log
}
return(result);
}
public override void OnStop()
{
this.telemetry.TrackTrace("WorkerRole stopping", SeverityLevel.Information);
try
{
this.stopEvent.Set();
if (this.settingsWatcher != null)
{
this.settingsWatcher.Dispose();
this.settingsWatcher = null;
}
if (this.trainProcesserHost != null)
{
this.trainProcesserHost.Dispose();
this.trainProcesserHost = null;
}
if (this.webApp != null)
{
this.webApp.Dispose();
this.webApp = null;
}
base.OnStop();
}
catch (Exception e)
{
this.telemetry.TrackException(e);
}
this.telemetry.TrackTrace("WorkerRole stopped", SeverityLevel.Information);
}
public void Dispose()
{
if (trainProcesserHost != null)
{
trainProcesserHost.Dispose();
trainProcesserHost = null;
}
}
internal OnlineTrainerSettingsWatcher(LearnEventProcessorHost trainProcessorHost)
{
this.telemetry = new TelemetryClient();
this.trainProcessorHost = trainProcessorHost;
RoleEnvironment.Changed += RoleEnvironment_Changed;
this.settingsDownloader = new OnlineTrainerSettingsDownloader(TimeSpan.FromSeconds(5));
this.settingsDownloader.Downloaded += AzureSettingsBlobDownloader_Downloaded;
this.settingsDownloader.Failed += AzureSettingsBlobDownloader_Failed;
}
internal async Task StartAsync(ICheckpointPolicy checkpointPolicy)
{
trainProcesserHost = new LearnEventProcessorHost();
await trainProcesserHost.StartAsync(new OnlineTrainerSettingsInternal
{
CheckpointPolicy = checkpointPolicy,
JoinedEventHubConnectionString = inputEventHubConnectionString,
EvalEventHubConnectionString = evalEventHubConnectionString,
StorageConnectionString = storageConnectionString,
Metadata = new OnlineTrainerSettings
{
ApplicationID = "vwunittest",
TrainArguments = trainArguments
},
EnableExampleTracing = false,
EventHubStartDateTimeUtc = DateTime.UtcNow // ignore any events that arrived before this time
});
AssertNoExceptionsThroughAppInsights();
}
public CheckpointController(LearnEventProcessorHost trainProcessorFactory)
: base(trainProcessorFactory)
{
}
public ResetController(LearnEventProcessorHost trainProcessorFactory)
: base(trainProcessorFactory)
{
}
public OnlineTrainerController(LearnEventProcessorHost trainProcessorFactory)
{
this.telemetry = new TelemetryClient();
this.trainProcessorHost = trainProcessorFactory;
this.adminToken = CloudConfigurationManager.GetSetting("AdminToken");
}
public async Task TestAzureTrainer()
{
var storageConnectionString = GetConfiguration("storageConnectionString");
var inputEventHubConnectionString = GetConfiguration("inputEventHubConnectionString");
var evalEventHubConnectionString = GetConfiguration("evalEventHubConnectionString");
var trainArguments = "--cb_explore_adf --epsilon 0.2 -q ab";
// register with AppInsights to collect exceptions
var exceptions = RegisterAppInsightExceptionHook();
// cleanup blobs
var blobs = new ModelBlobs(storageConnectionString);
await blobs.Cleanup();
var data = GenerateData(100).ToDictionary(d => d.EventId, d => d);
// start listening for event hub
using (var trainProcesserHost = new LearnEventProcessorHost())
{
await trainProcesserHost.StartAsync(new OnlineTrainerSettingsInternal
{
CheckpointPolicy = new CountingCheckpointPolicy(data.Count),
JoinedEventHubConnectionString = inputEventHubConnectionString,
EvalEventHubConnectionString = evalEventHubConnectionString,
StorageConnectionString = storageConnectionString,
Metadata = new OnlineTrainerSettings
{
ApplicationID = "vwunittest",
TrainArguments = trainArguments
},
EnableExampleTracing = true,
EventHubStartDateTimeUtc = DateTime.UtcNow // ignore any events that arrived before this time
});
// send events to event hub
var eventHubInputClient = EventHubClient.CreateFromConnectionString(inputEventHubConnectionString);
data.Values.ForEach(c => eventHubInputClient.Send(new EventData(c.JSONAsBytes)
{
PartitionKey = c.Index.ToString()
}));
// wait for trainer to checkpoint
await blobs.PollTrainerCheckpoint(exceptions);
// download & parse trackback file
var trackback = blobs.DownloadTrackback();
Assert.AreEqual(data.Count, trackback.EventIds.Count);
// train model offline using trackback
var settings = new VowpalWabbitSettings(trainArguments + $" --id {trackback.ModelId} --save_resume --readable_model offline.json.model.txt -f offline.json.model");
using (var vw = new VowpalWabbitJson(settings))
{
foreach (var id in trackback.EventIds)
{
var json = data[id].JSON;
var progressivePrediction = vw.Learn(json, VowpalWabbitPredictionType.ActionProbabilities);
// TODO: validate eval output
}
vw.Native.SaveModel("offline.json.2.model");
}
// download online model
new CloudBlob(blobs.ModelBlob.Uri, blobs.BlobClient.Credentials).DownloadToFile("online.model", FileMode.Create);
// validate that the model is the same
CollectionAssert.AreEqual(
File.ReadAllBytes("offline.json.model"),
File.ReadAllBytes("online.model"),
"Offline and online model differs. Run to 'vw -i online.model --readable_model online.model.txt' to compare");
}
}
public StatusController(LearnEventProcessorHost trainProcessorHost)
{
this.trainProcessorHost = trainProcessorHost;
}
