private void SaveTrainSession(Guid trainSessionId, TrainSessionState sessionState,
bool inUse = false, ModelSummary modelSummary = null, string errorMessage = null)
{
Guid mlModelStateId = TrainSessionStateMapping[sessionState];
var updateQuery = (Update) new Update(_userConnection, "MLTrainSession")
.Set("StateId", Column.Const(mlModelStateId))
.Set("Error", Column.Parameter(errorMessage ?? string.Empty))
.Set("TrainedOn", Column.Parameter(DateTime.UtcNow))
.Where("Id").IsEqual(Column.Parameter(trainSessionId));
if (inUse)
{
updateQuery = updateQuery.Set("InUse", Column.Parameter(true));
}
if (modelSummary != null)
{
string serializedFeatureImportances = JsonConvert.SerializeObject(modelSummary.FeatureImportances);
updateQuery = updateQuery
.Set("TrainSetSize", Column.Const(modelSummary.DataSetSize))
.Set("InstanceMetric", Column.Parameter(modelSummary.Metric))
.Set("TrainingTimeMinutes", Column.Const(modelSummary.TrainingTimeMinutes))
.Set("FeatureImportances", Column.Parameter(serializedFeatureImportances));
}
updateQuery.Execute();
}
/// <summary>
/// Queries for the actual model state and updates it in database.
/// </summary>
public void UpdateModelState()
{
Guid modelId = _modelConfig.Id;
Guid sessionId = _modelConfig.TrainSessionId;
sessionId.CheckArgumentEmpty("MLModel.TrainSessionId");
GetSessionInfoResponse response;
try {
response = _proxy.GetTrainingSessionInfo(sessionId);
} catch (Exception e) {
UpdateModelOnError(_modelConfig.Id, e.Message);
throw;
}
TrainSessionState newSessionState = response.SessionState;
_log.Info($"For model {modelId} new model state - {newSessionState}, old was {_modelConfig.CurrentState}");
if (FiniteStates.Contains(newSessionState))
{
UpdateTriedToTrainOn(modelId);
}
if (newSessionState == TrainSessionState.Done)
{
try {
ModelSummary modelSummary = response.ModelSummary;
var ignoresMetric = GetIsTrainSessionIgnoresMetric(sessionId);
if (ignoresMetric || GetIsModelAcceptable(response))
{
UpdateModelInstance(response, sessionId);
SaveAcceptedSession(modelId, sessionId, response.ModelSummary);
_log.Info("\tModel was successfully trained. " +
$"Instance uid: {modelSummary.ModelInstanceUId}, instance metric: {modelSummary.Metric}");
}
else
{
string message = "\tModel for schema {0} was successfully trained, but is not acceptable " +
"to be applied. Instance uid: {1}, instance metric: {2}, metric threshold: {3}";
_log.WarnFormat(message, _modelConfig.Id, modelSummary.ModelInstanceUId, modelSummary.Metric,
_modelConfig.MetricThreshold);
SaveUnacceptedSession(sessionId, modelSummary);
}
} catch (Exception ex) {
UpdateModelOnError(modelId, ex.Message);
SaveFailedSession(sessionId, ex.Message);
throw;
}
}
UpdateTrainSessionStatus(newSessionState, modelId, response.ErrorMessage);
SaveTrainSession(_modelConfig.TrainSessionId, newSessionState, errorMessage: response.ErrorMessage);
}
private void InsertTrainSession(Guid mlModelId, Guid trainSessionId, TrainSessionState sessionState,
bool ignoreMetricThreshold, string errorMessage = null)
{
Guid mlModelStateId = TrainSessionStateMapping[sessionState];
var insertQuery = new Insert(_userConnection).Into("MLTrainSession")
.Set("Id", Column.Const(trainSessionId))
.Set("MLModelId", Column.Const(mlModelId))
.Set("StateId", Column.Const(mlModelStateId))
.Set("InUse", Column.Parameter(false))
.Set("IgnoreMetricThreshold", Column.Parameter(ignoreMetricThreshold))
.Set("Error", Column.Parameter(errorMessage ?? string.Empty));
insertQuery.Execute();
}
private void NotifyUsers(TrainSessionState newSessionState, Guid modelId)
{
EntitySchema schema = _userConnection.EntitySchemaManager.GetInstanceByName("MLModel");
Entity model = schema.CreateEntity(_userConnection);
if (!model.FetchFromDB("Id", modelId, new[] { schema.PrimaryDisplayColumn.Name }))
{
return;
}
LocalizableString template = new LocalizableString(_userConnection.Workspace.ResourceStorage,
"MLModelTrainer", "LocalizableStrings.ModelStateChangedMessage.Value");
string message = string.Format(template, model.PrimaryDisplayColumnValue, newSessionState);
_modelEventsNotifier.NotifyModelEventRecipients(message);
}
/// <summary>
/// Updates the train session status.
/// </summary>
/// <param name="newSessionState">New state of the train session.</param>
/// <param name="modelId">The model's identifier.</param>
/// <param name="errorMessage">Optional error message to save into MLModel entity.</param>
protected virtual void UpdateTrainSessionStatus(TrainSessionState newSessionState, Guid modelId,
string errorMessage = "")
{
if (_modelConfig.CurrentState == newSessionState && errorMessage.IsNullOrEmpty())
{
return;
}
if (!TrainSessionStateMapping.ContainsKey(newSessionState))
{
throw new ItemNotFoundException($"Session state {newSessionState} is unknown");
}
Guid mlModelStateId = TrainSessionStateMapping[newSessionState];
Update updateQuery = (Update) new Update(_userConnection, "MLModel")
.Set("StateId", Column.Parameter(mlModelStateId))
.Set("LastError", Column.Parameter(errorMessage ?? string.Empty))
.Where("Id").IsEqual(Column.Parameter(modelId, "Guid"));
updateQuery.Execute();
_modelConfig.CurrentState = newSessionState;
if (_notificationStates.Contains(newSessionState))
{
NotifyUsers(newSessionState, modelId);
}
}
