public Response QueryLastTrainingState(Guid modelId) { modelId.CheckArgumentEmpty(nameof(modelId)); var modelConfig = GetModelConfig(modelId); var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig); var userConnectionArg = new ConstructorArgument("userConnection", UserConnection); IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg); TrainSessionState[] notInProgressStates = { TrainSessionState.NotStarted, TrainSessionState.Done, TrainSessionState.Error }; if (!notInProgressStates.Contains(modelConfig.CurrentState)) { if (modelConfig.TrainSessionId.IsEmpty()) { var message = $"Model {modelId} has state {modelConfig.CurrentState}, but TrainSessionId is empty"; _log.Error(message); throw new InvalidObjectStateException(message); } trainer.UpdateModelState(); } return(new Response { Value = modelConfig.CurrentState.ToString() }); }
private static void QueryTrainingState(IMLModelTrainer trainer, Guid modelId) { try { trainer.UpdateModelState(); } catch (Exception e) { _log.ErrorFormat("Error occurred while trying to update training state for ML model {0}", e, modelId); } }
private static void Train(IMLModelTrainer trainer, Guid modelId, bool ignoreMetricThreshold = false) { _log.InfoFormat("Starting session for model id '{0}'", modelId); Guid sessionId = trainer.StartTrainSession(ignoreMetricThreshold); _log.InfoFormat("Uploading data for session '{0}'", sessionId); trainer.UploadData(); _log.InfoFormat("Begin training for session '{0}'", sessionId); trainer.BeginTraining(); _log.InfoFormat("Training for session '{0}' is queued", sessionId); }
/// <summary> /// Processes ML models, that need to be trained due to maintaince window. /// </summary> /// <param name="userConnection">User connection.</param> public virtual void ProcessAllModels(UserConnection userConnection) { _log.Info("Start to process (training, state sync) ML models"); if (!MLUtils.CheckIsServiceUrlSet(userConnection)) { return; } if (!userConnection.LicHelper.GetHasOperationLicense(MLConsts.LicOperationCode)) { _log.Error($"License for operation '{MLConsts.LicOperationCode}' is missing"); return; } if (!MLUtils.CheckApiKey(userConnection)) { return; } IList <MLModelConfig> modelsToStartTrain = GetModelsNeedToBeTrained(userConnection); List <MLModelConfig> modelsToReceiveTrainingResult = GetModelsInTrainingStatus(userConnection); var userConnectionArg = new ConstructorArgument("userConnection", userConnection); foreach (MLModelConfig modelConfig in modelsToStartTrain) { try { var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig); IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg); Train(trainer, modelConfig.Id); } catch (Exception e) { _log.Error($"Error occurred while trying to begin training for ML model {modelConfig.Id}", e); } } foreach (MLModelConfig modelConfig in modelsToReceiveTrainingResult) { var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig); IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg); QueryTrainingState(trainer, modelConfig.Id); } _log.Info("ML model's processing finished"); }
/// <summary> /// Processes ML model, that has to be trained. /// </summary> /// <param name="userConnection">User connection.</param> /// <param name="modelId">Model's identifier.</param> /// <param name="ignoreMetricThreshold">Optional mode, that force success training result to be applied as /// working.</param> /// <returns></returns> /// <exception cref="IncorrectConfigurationException">CloudServicesAPIKey</exception> public virtual TrainSessionState ProcessModel(UserConnection userConnection, Guid modelId, bool ignoreMetricThreshold = false) { userConnection.LicHelper.CheckHasOperationLicense(MLConsts.LicOperationCode); if (!MLUtils.CheckApiKey(userConnection)) { throw new IncorrectConfigurationException("CloudServicesAPIKey"); } MLModelConfig modelConfig = GetModelConfig(userConnection, modelId); var modelConfigArg = new ConstructorArgument("modelConfig", modelConfig); var userConnectionArg = new ConstructorArgument("userConnection", userConnection); IMLModelTrainer trainer = ClassFactory.Get <IMLModelTrainer>(userConnectionArg, modelConfigArg); TrainSessionState[] notInProgressStates = { TrainSessionState.NotStarted, TrainSessionState.Done, TrainSessionState.Error }; if (notInProgressStates.Contains(modelConfig.CurrentState)) { Train(trainer, modelConfig.Id, ignoreMetricThreshold); } QueryTrainingState(trainer, modelConfig.Id); return(modelConfig.CurrentState); }