public Response QueryLastTrainingState(Guid modelId)
{
modelId.CheckArgumentEmpty(nameof(modelId));
var modelConfig = GetModelConfig(modelId);
var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig);
var userConnectionArg = new ConstructorArgument("userConnection", UserConnection);
IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg);
TrainSessionState[] notInProgressStates =
{
TrainSessionState.NotStarted, TrainSessionState.Done, TrainSessionState.Error
};
if (!notInProgressStates.Contains(modelConfig.CurrentState))
{
if (modelConfig.TrainSessionId.IsEmpty())
{
var message = $"Model {modelId} has state {modelConfig.CurrentState}, but TrainSessionId is empty";
_log.Error(message);
throw new InvalidObjectStateException(message);
}
trainer.UpdateModelState();
}
return(new Response {
Value = modelConfig.CurrentState.ToString()
});
}
private static void QueryTrainingState(IMLModelTrainer trainer, Guid modelId)
{
try {
trainer.UpdateModelState();
} catch (Exception e) {
_log.ErrorFormat("Error occurred while trying to update training state for ML model {0}", e, modelId);
}
}
private static void Train(IMLModelTrainer trainer, Guid modelId, bool ignoreMetricThreshold = false)
{
_log.InfoFormat("Starting session for model id '{0}'", modelId);
Guid sessionId = trainer.StartTrainSession(ignoreMetricThreshold);
_log.InfoFormat("Uploading data for session '{0}'", sessionId);
trainer.UploadData();
_log.InfoFormat("Begin training for session '{0}'", sessionId);
trainer.BeginTraining();
_log.InfoFormat("Training for session '{0}' is queued", sessionId);
}
/// <summary>
/// Processes ML models, that need to be trained due to maintaince window.
/// </summary>
/// <param name="userConnection">User connection.</param>
public virtual void ProcessAllModels(UserConnection userConnection)
{
_log.Info("Start to process (training, state sync) ML models");
if (!MLUtils.CheckIsServiceUrlSet(userConnection))
{
return;
}
if (!userConnection.LicHelper.GetHasOperationLicense(MLConsts.LicOperationCode))
{
_log.Error($"License for operation '{MLConsts.LicOperationCode}' is missing");
return;
}
if (!MLUtils.CheckApiKey(userConnection))
{
return;
}
IList <MLModelConfig> modelsToStartTrain = GetModelsNeedToBeTrained(userConnection);
List <MLModelConfig> modelsToReceiveTrainingResult = GetModelsInTrainingStatus(userConnection);
var userConnectionArg = new ConstructorArgument("userConnection", userConnection);
foreach (MLModelConfig modelConfig in modelsToStartTrain)
{
try {
var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig);
IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg);
Train(trainer, modelConfig.Id);
} catch (Exception e) {
_log.Error($"Error occurred while trying to begin training for ML model {modelConfig.Id}", e);
}
}
foreach (MLModelConfig modelConfig in modelsToReceiveTrainingResult)
{
var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig);
IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg);
QueryTrainingState(trainer, modelConfig.Id);
}
_log.Info("ML model's processing finished");
}
/// <summary>
/// Processes ML model, that has to be trained.
/// </summary>
/// <param name="userConnection">User connection.</param>
/// <param name="modelId">Model's identifier.</param>
/// <param name="ignoreMetricThreshold">Optional mode, that force success training result to be applied as
/// working.</param>
/// <returns></returns>
/// <exception cref="IncorrectConfigurationException">CloudServicesAPIKey</exception>
public virtual TrainSessionState ProcessModel(UserConnection userConnection, Guid modelId,
bool ignoreMetricThreshold = false)
{
userConnection.LicHelper.CheckHasOperationLicense(MLConsts.LicOperationCode);
if (!MLUtils.CheckApiKey(userConnection))
{
throw new IncorrectConfigurationException("CloudServicesAPIKey");
}
MLModelConfig modelConfig = GetModelConfig(userConnection, modelId);
var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig);
var userConnectionArg = new ConstructorArgument("userConnection", userConnection);
IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg);
TrainSessionState[] notInProgressStates =
{
TrainSessionState.NotStarted, TrainSessionState.Done, TrainSessionState.Error
};
if (notInProgressStates.Contains(modelConfig.CurrentState))
{
Train(trainer, modelConfig.Id, ignoreMetricThreshold);
}
QueryTrainingState(trainer, modelConfig.Id);
return(modelConfig.CurrentState);
}
