/// <summary>
/// 引数の前処理インスタンスに、入力モデルの値を詰め込む。
/// 成功時はnullを返す。エラーが発生したらエラー内容を返す。
/// 事前に<see cref="CreateInputModel.GitModel"/>の入力チェックを行っておくこと。
/// </summary>
private async Task <IActionResult> SetPreprocessDetailsAsync(Preprocess preprocessing, CreateInputModel model)
{
long? gitId = null;
string repository = null;
string owner = null;
string branch = null;
string commitId = null;
if (model.GitModel != null)
{
gitId = model.GitModel.GitId ?? CurrentUserInfo.SelectedTenant.DefaultGit?.Id;
repository = model.GitModel.Repository;
owner = model.GitModel.Owner;
branch = model.GitModel.Branch;
commitId = model.GitModel.CommitId;
//コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(commitId))
{
commitId = await gitLogic.GetCommitIdAsync(gitId.Value, model.GitModel.Repository, model.GitModel.Owner, model.GitModel.Branch);
if (string.IsNullOrEmpty(commitId))
{
//コミットIDが特定できなかったらエラー
return(JsonNotFound($"The branch {branch} for {gitId.Value}/{model.GitModel.Owner}/{model.GitModel.Repository} is not found."));
}
}
}
long? registryId = null;
string image = null;
string tag = null;
if (model.ContainerImage != null)
{
registryId = model.ContainerImage.RegistryId ?? CurrentUserInfo.SelectedTenant.DefaultRegistry?.Id;
image = model.ContainerImage.Image;
tag = model.ContainerImage.Tag;
}
preprocessing.Name = model.Name;
preprocessing.EntryPoint = model.EntryPoint;
preprocessing.ContainerRegistryId = registryId;
preprocessing.ContainerImage = image;
preprocessing.ContainerTag = tag; //latestは運用上使用されていないハズなので、そのまま直接代入
preprocessing.RepositoryGitId = gitId;
preprocessing.RepositoryName = repository;
preprocessing.RepositoryOwner = owner;
preprocessing.RepositoryBranch = branch;
preprocessing.RepositoryCommitId = commitId;
preprocessing.Memo = model.Memo;
preprocessing.Cpu = model.Cpu;
preprocessing.Memory = model.Memory;
preprocessing.Gpu = model.Gpu;
return(null);
}
public async Task <IActionResult> Create([FromBody] CreateInputModel model, [FromServices] INodeRepository nodeRepository)
{
//データの入力チェック
if (!ModelState.IsValid)
{
return(JsonBadRequest("Invalid inputs."));
}
//データセットが指定されていれば存在チェック
if (model.DataSetId.HasValue)
{
var dataSet = await dataSetRepository.GetByIdAsync(model.DataSetId.Value);
if (dataSet == null)
{
return(JsonNotFound($"DataSet ID {model.DataSetId} is not found."));
}
}
if (string.IsNullOrEmpty(model.Partition) == false)
{
bool existPartition = await nodeRepository.IsEnablePartitionAsync(model.Partition, true);
if (existPartition == false)
{
return(JsonNotFound($"There are no enable nodes with Partition {model.Partition}."));
}
}
// 環境変数名のチェック
if (model.Options != null && model.Options.Count > 0)
{
foreach (var env in model.Options)
{
if (!string.IsNullOrEmpty(env.Key))
{
// フォーマットチェック
if (!Regex.IsMatch(env.Key, "^[-._a-zA-Z][-._a-zA-Z0-9]*$"))
{
return(JsonNotFound($"Invalid envName. Please match the format of '^[-._a-zA-Z][-._a-zA-Z0-9]*$'."));
}
}
}
}
//コンテナの実行前に、ノートブック履歴を作成する（コンテナの実行に失敗した場合、そのステータスをユーザに表示するため）
var notebookHistory = new NotebookHistory()
{
DisplayId = -1,
Name = model.Name,
DataSetId = model.DataSetId,
OptionDic = model.Options ?? new Dictionary <string, string>(), //オプションはnullの可能性があるので、その時は初期化
Cpu = model.Cpu.Value,
Memory = model.Memory.Value,
Gpu = model.Gpu.Value,
Partition = model.Partition,
Memo = model.Memo,
Status = ContainerStatus.Running.Key,
StartedAt = DateTime.Now,
ExpiresIn = model.ExpiresIn
};
//コンテナが指定されているかチェック
if (model.ContainerImage != null)
{
notebookHistory.ContainerRegistryId = model.ContainerImage.RegistryId ?? CurrentUserInfo.SelectedTenant.DefaultRegistry?.Id;
notebookHistory.ContainerImage = model.ContainerImage.Image;
notebookHistory.ContainerTag = model.ContainerImage.Tag; //latestは運用上使用されていないハズなので、そのまま直接代入
}
else
{
//コンテナイメージの設定がない場合デフォルトのイメージを設定
notebookHistory.ContainerRegistryId = null;
notebookHistory.ContainerImage = "tensorflow/tensorflow";
notebookHistory.ContainerTag = "1.13.1-gpu-py3";
}
//gitが指定されているかチェック
if (model.GitModel != null)
{
//gitリポジトリ名が指定されていれば、ブランチ、コミットIDを設定。指定されていなければnull
long? gitId = model.GitModel.GitId ?? CurrentUserInfo.SelectedTenant.DefaultGit?.Id;
string branch = null;
string commitId = null;
if (!string.IsNullOrEmpty(model.GitModel.Repository))
{
branch = model.GitModel.Branch ?? "master";
commitId = model.GitModel.CommitId;
//コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(commitId))
{
commitId = await gitLogic.GetCommitIdAsync(gitId.Value, model.GitModel.Repository, model.GitModel.Owner, branch);
if (string.IsNullOrEmpty(commitId))
{
//コミットIDが特定できなかったらエラー
return(JsonNotFound($"The branch {branch} for {gitId.Value}/{model.GitModel.Owner}/{model.GitModel.Repository} is not found."));
}
}
}
//git情報を設定
notebookHistory.ModelGitId = gitId.Value;
notebookHistory.ModelRepository = model.GitModel.Repository;
notebookHistory.ModelRepositoryOwner = model.GitModel.Owner;
notebookHistory.ModelBranch = branch;
notebookHistory.ModelCommitId = commitId;
}
if (notebookHistory.OptionDic.ContainsKey("")) //空文字は除外する
{
notebookHistory.OptionDic.Remove("");
}
// 親学習が指定されていれば存在チェック
if (model.ParentIds != null)
{
var maps = new List <NotebookHistoryParentTrainingMap>();
foreach (var parentId in model.ParentIds)
{
var parent = await trainingHistoryRepository.GetByIdAsync(parentId);
if (parent == null)
{
return(JsonNotFound($"Training ID {parentId} is not found."));
}
// ノートブック履歴に親学習を紐づける
var map = notebookHistoryRepository.AttachParentToNotebookAsync(notebookHistory, parent);
if (map != null)
{
maps.Add(map);
}
}
notebookHistory.ParentTrainingMaps = maps;
}
notebookHistoryRepository.Add(notebookHistory);
unitOfWork.Commit();
var result = await clusterManagementLogic.RunNotebookContainerAsync(notebookHistory);
if (result.IsSuccess == false)
{
//コンテナの起動に失敗した状態。エラーを出力して、保存したノートブック履歴も削除する。
notebookHistoryRepository.Delete(notebookHistory);
unitOfWork.Commit();
return(JsonError(HttpStatusCode.ServiceUnavailable, "Failed to run notebook. The message bellow may be help to resolve: " + result.Error));
}
//結果に従い、ノートブック結果を更新する。
//実行には時間がかかりうるので、DBから最新の情報を取ってくる
notebookHistory = await notebookHistoryRepository.GetByIdAsync(notebookHistory.Id);
notebookHistory.Configuration = result.Value.Configuration;
notebookHistory.Status = result.Value.Status.Key;
unitOfWork.Commit();
if (result.Value.Status.Succeed())
{
return(JsonCreated(new SimpleOutputModel(notebookHistory)));
}
else
{
return(JsonError(HttpStatusCode.ServiceUnavailable, $"Failed to run notebook. Status={result.Value.Status.Name}. Please contact your server administrator."));
}
}
public async Task <IActionResult> Create([FromBody] CreateInputModel model, [FromServices] INodeRepository nodeRepository)
{
//データの入力チェック
if (!ModelState.IsValid)
{
return(JsonBadRequest("Invalid inputs."));
}
//データの存在チェック
var dataSet = await dataSetRepository.GetByIdAsync(model.DataSetId.Value);
if (dataSet == null)
{
return(JsonNotFound($"DataSet ID {model.DataSetId} is not found."));
}
if (model.ParentId.HasValue)
{
var parent = await trainingHistoryRepository.GetByIdAsync(model.ParentId.Value);
if (parent == null)
{
return(JsonNotFound($"Training ID {model.ParentId.Value} is not found."));
}
}
if (string.IsNullOrEmpty(model.Partition) == false)
{
bool existPartition = await nodeRepository.IsEnablePartitionAsync(model.Partition, true);
if (existPartition == false)
{
return(JsonNotFound($"There are no enable nodes with Partition {model.Partition}."));
}
}
// 環境変数名のチェック
if (model.Options != null && model.Options.Count > 0)
{
foreach (var env in model.Options)
{
if (!string.IsNullOrEmpty(env.Key))
{
// フォーマットチェック
if (!Regex.IsMatch(env.Key, "^[-._a-zA-Z][-._a-zA-Z0-9]*$"))
{
return(JsonNotFound($"Invalid envName. Please match the format of '^[-._a-zA-Z][-._a-zA-Z0-9]*$'."));
}
}
}
}
long? gitId = model.GitModel.GitId ?? CurrentUserInfo.SelectedTenant.DefaultGit?.Id;
string branch = model.GitModel.Branch ?? "master";
string commitId = model.GitModel.CommitId;
//コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(commitId))
{
commitId = await gitLogic.GetCommitIdAsync(gitId.Value, model.GitModel.Repository, model.GitModel.Owner, branch);
if (string.IsNullOrEmpty(commitId))
{
//コミットIDが特定できなかったらエラー
return(JsonNotFound($"The branch {branch} for {gitId.Value}/{model.GitModel.Owner}/{model.GitModel.Repository} is not found."));
}
}
//コンテナの実行前に、学習履歴を作成する（コンテナの実行に失敗した場合、そのステータスをユーザに表示するため）
var trainingHistory = new TrainingHistory()
{
Name = model.Name,
DisplayId = -1,
ContainerRegistryId = model.ContainerImage.RegistryId ?? CurrentUserInfo.SelectedTenant.DefaultRegistry?.Id,
ContainerImage = model.ContainerImage.Image,
ContainerTag = model.ContainerImage.Tag, //latestは運用上使用されていないハズなので、そのまま直接代入
DataSetId = model.DataSetId.Value,
EntryPoint = model.EntryPoint,
ModelGitId = gitId.Value,
ModelRepository = model.GitModel.Repository,
ModelRepositoryOwner = model.GitModel.Owner,
ModelBranch = branch,
ModelCommitId = commitId,
OptionDic = model.Options ?? new Dictionary <string, string>(), //オプションはnullの可能性があるので、その時は初期化
ParentId = model.ParentId,
Memo = model.Memo,
Cpu = model.Cpu.Value,
Memory = model.Memory.Value,
Gpu = model.Gpu.Value,
Partition = model.Partition,
Status = ContainerStatus.Running.Key
};
if (trainingHistory.OptionDic.ContainsKey("")) //空文字は除外する
{
trainingHistory.OptionDic.Remove("");
}
trainingHistoryRepository.Add(trainingHistory);
if (dataSet.IsLocked == false)
{
dataSet.IsLocked = true;
}
unitOfWork.Commit();
var result = await clusterManagementLogic.RunTrainContainerAsync(trainingHistory);
if (result.IsSuccess == false)
{
//コンテナの起動に失敗した状態。エラーを出力して、保存した学習履歴も削除する。
trainingHistoryRepository.Delete(trainingHistory);
unitOfWork.Commit();
return(JsonError(HttpStatusCode.ServiceUnavailable, "Failed to run training. The message bellow may be help to resolve: " + result.Error));
}
//結果に従い、学習結果を更新する。
//実行には時間がかかりうるので、DBから最新の情報を取ってくる
trainingHistory = await trainingHistoryRepository.GetByIdAsync(trainingHistory.Id);
trainingHistory.Configuration = result.Value.Configuration;
trainingHistory.Status = result.Value.Status.Key;
unitOfWork.Commit();
if (result.Value.Status.Succeed())
{
return(JsonCreated(new SimpleOutputModel(trainingHistory)));
}
else
{
return(JsonError(HttpStatusCode.ServiceUnavailable, $"Failed to run training. Status={result.Value.Status.Name}. Please contact your server administrator."));
}
}
private async Task <IActionResult> SetTemplateDetails(TemplateVersion templateVersion, Template template, VersionCreateInputModel model)
{
long? preprocessGitId = null;
string preprocessRepository = null;
string preprocessOwner = null;
string preprocessBranch = null;
string preprocessCommitId = null;
string preprocessGitToken = null;
long trainingGitId = 0;
string trainingRepository = null;
string trainingOwner = null;
string trainingBranch = null;
string trainingCommitId = null;
string trainingGitToken = null;
long? evaluationGitId = null;
string evaluationRepository = null;
string evaluationOwner = null;
string evaluationBranch = null;
string evaluationCommitId = null;
string evaluationGitToken = null;
if (model.PreprocessGitModel != null)
{
preprocessGitId = model.PreprocessGitModel.GitId ?? CurrentUserInfo.SelectedTenant.DefaultGit?.Id;
preprocessRepository = model.PreprocessGitModel.Repository;
preprocessOwner = model.PreprocessGitModel.Owner;
preprocessBranch = model.PreprocessGitModel.Branch;
preprocessCommitId = model.PreprocessGitModel.CommitId;
// コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(preprocessCommitId))
{
preprocessCommitId = await gitLogic.GetCommitIdAsync(preprocessGitId.Value, model.PreprocessGitModel.Repository, model.PreprocessGitModel.Owner, model.PreprocessGitModel.Branch);
if (string.IsNullOrEmpty(preprocessCommitId))
{
// コミットIDが特定できなかったらエラー
return(JsonNotFound($"The branch {preprocessBranch} for {preprocessGitId.Value}/{model.PreprocessGitModel.Owner}/{ model.PreprocessGitModel.Repository} is not found."));
}
}
preprocessGitToken = model.PreprocessGitModel.Token;
}
if (model.TrainingGitModel != null)
{
trainingGitId = (model.TrainingGitModel.GitId ?? CurrentUserInfo.SelectedTenant.DefaultGit?.Id).Value;
trainingRepository = model.TrainingGitModel.Repository;
trainingOwner = model.TrainingGitModel.Owner;
trainingBranch = model.TrainingGitModel.Branch;
trainingCommitId = model.TrainingGitModel.CommitId;
// コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(trainingCommitId))
{
trainingCommitId = await gitLogic.GetCommitIdAsync(trainingGitId, model.TrainingGitModel.Repository, model.TrainingGitModel.Owner, model.TrainingGitModel.Branch);
if (string.IsNullOrEmpty(trainingCommitId))
{
// コミットIDが特定できなかったらエラー
return(JsonNotFound($"The branch {trainingBranch} for {trainingGitId}/{model.TrainingGitModel.Owner}/{ model.TrainingGitModel.Repository} is not found."));
}
}
trainingGitToken = model.TrainingGitModel.Token;
}
if (model.EvaluationGitModel != null)
{
evaluationGitId = (model.EvaluationGitModel.GitId ?? CurrentUserInfo.SelectedTenant.DefaultGit?.Id).Value;
evaluationRepository = model.EvaluationGitModel.Repository;
evaluationOwner = model.EvaluationGitModel.Owner;
evaluationBranch = model.EvaluationGitModel.Branch;
evaluationCommitId = model.EvaluationGitModel.CommitId;
// コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(evaluationCommitId))
{
evaluationCommitId = await gitLogic.GetCommitIdAsync(evaluationGitId.Value, model.EvaluationGitModel.Repository, model.EvaluationGitModel.Owner, model.EvaluationGitModel.Branch);
if (string.IsNullOrEmpty(evaluationCommitId))
{
// コミットIDが特定できなかったらエラー
return(JsonNotFound($"The branch {evaluationBranch} for {evaluationGitId}/{model.EvaluationGitModel.Owner}/{ model.EvaluationGitModel.Repository} is not found."));
}
}
evaluationGitToken = model.EvaluationGitModel.Token;
}
long? preprocessRegistryId = null;
string preprocessImage = null;
string preprocessTag = null;
string preprocessRegistryToken = null;
if (model.PreprocessContainerImage != null)
{
preprocessRegistryId = model.PreprocessContainerImage.RegistryId ?? CurrentUserInfo.SelectedTenant.DefaultRegistry?.Id;
preprocessImage = model.PreprocessContainerImage.Image;
preprocessTag = model.PreprocessContainerImage.Tag;
preprocessRegistryToken = model.PreprocessContainerImage.Token;
}
long trainingRegistryId = 0;
string trainingImage = null;
string trainingTag = null;
string trainingRegistryToken = null;
if (model.TrainingContainerImage != null)
{
trainingRegistryId = (model.TrainingContainerImage.RegistryId ?? CurrentUserInfo.SelectedTenant.DefaultRegistry?.Id).Value;
trainingImage = model.TrainingContainerImage.Image;
trainingTag = model.TrainingContainerImage.Tag;
trainingRegistryToken = model.TrainingContainerImage.Token;
}
long? evaluationRegistryId = null;
string evaluationImage = null;
string evaluationTag = null;
string evaluationRegistryToken = null;
if (model.EvaluationContainerImage != null)
{
evaluationRegistryId = (model.EvaluationContainerImage.RegistryId ?? CurrentUserInfo.SelectedTenant.DefaultRegistry?.Id).Value;
evaluationImage = model.EvaluationContainerImage.Image;
evaluationTag = model.EvaluationContainerImage.Tag;
evaluationRegistryToken = model.EvaluationContainerImage.Token;
}
templateVersion.TemplateId = template.Id;
templateVersion.Version = template.LatestVersion;
templateVersion.PreprocessEntryPoint = model.PreprocessEntryPoint;
templateVersion.PreprocessContainerRegistryId = preprocessRegistryId;
templateVersion.PreprocessContainerImage = preprocessImage;
templateVersion.PreprocessContainerTag = preprocessTag; // latestは運用上使用されていないハズなので、そのまま直接代入
templateVersion.PreprocessContainerToken = preprocessRegistryToken;
templateVersion.PreprocessRepositoryGitId = preprocessGitId;
templateVersion.PreprocessRepositoryName = preprocessRepository;
templateVersion.PreprocessRepositoryOwner = preprocessOwner;
templateVersion.PreprocessRepositoryBranch = preprocessBranch;
templateVersion.PreprocessRepositoryCommitId = preprocessCommitId;
templateVersion.PreprocessRepositoryToken = preprocessGitToken;
templateVersion.PreprocessCpu = model.PreprocessCpu;
templateVersion.PreprocessMemory = model.PreprocessMemory;
templateVersion.PreprocessGpu = model.PreprocessGpu;
templateVersion.TrainingEntryPoint = model.TrainingEntryPoint;
templateVersion.TrainingContainerRegistryId = trainingRegistryId;
templateVersion.TrainingContainerImage = trainingImage;
templateVersion.TrainingContainerTag = trainingTag; // latestは運用上使用されていないハズなので、そのまま直接代入
templateVersion.TrainingContainerToken = trainingRegistryToken;
templateVersion.TrainingRepositoryGitId = trainingGitId;
templateVersion.TrainingRepositoryName = trainingRepository;
templateVersion.TrainingRepositoryOwner = trainingOwner;
templateVersion.TrainingRepositoryBranch = trainingBranch;
templateVersion.TrainingRepositoryCommitId = trainingCommitId;
templateVersion.TrainingRepositoryToken = trainingGitToken;
templateVersion.TrainingCpu = model.TrainingCpu;
templateVersion.TrainingMemory = model.TrainingMemory;
templateVersion.TrainingGpu = model.TrainingGpu;
templateVersion.EvaluationEntryPoint = model.EvaluationEntryPoint;
templateVersion.EvaluationContainerRegistryId = evaluationRegistryId;
templateVersion.EvaluationContainerImage = evaluationImage;
templateVersion.EvaluationContainerTag = evaluationTag; // latestは運用上使用されていないハズなので、そのまま直接代入
templateVersion.EvaluationContainerToken = evaluationRegistryToken;
templateVersion.EvaluationRepositoryGitId = evaluationGitId;
templateVersion.EvaluationRepositoryName = evaluationRepository;
templateVersion.EvaluationRepositoryOwner = evaluationOwner;
templateVersion.EvaluationRepositoryBranch = evaluationBranch;
templateVersion.EvaluationRepositoryCommitId = evaluationCommitId;
templateVersion.EvaluationRepositoryToken = evaluationGitToken;
templateVersion.EvaluationCpu = model.EvaluationCpu;
templateVersion.EvaluationMemory = model.EvaluationMemory;
templateVersion.EvaluationGpu = model.EvaluationGpu;
return(null);
}
public static async Task <(TrainingHistory, IActionResult)> DoCreate(CreateInputModel model,
IDataSetRepository dataSetRepository,
INodeRepository nodeRepository,
ITenantRepository tenantRepository,
ITrainingHistoryRepository trainingHistoryRepository,
IClusterManagementLogic clusterManagementLogic,
IDataSetLogic dataSetLogic,
IGitLogic gitLogic,
ITagLogic tagLogic,
IUnitOfWork unitOfWork,
UserInfo currentUserInfo,
ModelStateDictionary modelState,
string requestUrl,
string scriptType,
string regisryTokenName,
string gitToken
)
{
//データの入力チェック
if (!modelState.IsValid)
{
return(null,
DoJsonBadRequest(typeof(TrainingController), requestUrl, modelState, "Invalid inputs."));
}
//データの存在チェック
var dataSet = await dataSetRepository.GetByIdAsync(model.DataSetId.Value);
if (dataSet == null)
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"DataSet ID {model.DataSetId} is not found."));
}
if (string.IsNullOrEmpty(model.Partition) == false)
{
bool existPartition = await nodeRepository.IsEnablePartitionAsync(model.Partition, true);
if (existPartition == false)
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"There are no enable nodes with Partition {model.Partition}."));
}
}
// 環境変数名のチェック
if (model.Options != null && model.Options.Count > 0)
{
foreach (var env in model.Options)
{
if (!string.IsNullOrEmpty(env.Key))
{
// フォーマットチェック
if (!Regex.IsMatch(env.Key, "^[-._a-zA-Z][-._a-zA-Z0-9]*$"))
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState, $"Invalid envName. Please match the format of '^[-._a-zA-Z][-._a-zA-Z0-9]*$'."));
}
}
}
}
long? gitId = model.GitModel.GitId ?? currentUserInfo.SelectedTenant.DefaultGit?.Id;
string branch = model.GitModel.Branch ?? "master";
string commitId = model.GitModel.CommitId;
//コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(commitId))
{
commitId = await gitLogic.GetCommitIdAsync(gitId.Value, model.GitModel.Repository, model.GitModel.Owner, branch);
if (string.IsNullOrEmpty(commitId))
{
//コミットIDが特定できなかったらエラー
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"The branch {branch} for {gitId.Value}/{model.GitModel.Owner}/{model.GitModel.Repository} is not found."));
}
}
// 各リソースの超過チェック
Tenant tenant = tenantRepository.Get(currentUserInfo.SelectedTenant.Id);
string errorMessage = clusterManagementLogic.CheckQuota(tenant, model.Cpu.Value, model.Memory.Value, model.Gpu.Value);
if (errorMessage != null)
{
return(null, DoJsonBadRequest(typeof(TrainingController), requestUrl, modelState, errorMessage));
}
//コンテナの実行前に、学習履歴を作成する（コンテナの実行に失敗した場合、そのステータスをユーザに表示するため）
var trainingHistory = new TrainingHistory()
{
Name = model.Name,
DisplayId = -1,
ContainerRegistryId = model.ContainerImage.RegistryId ?? currentUserInfo.SelectedTenant.DefaultRegistry?.Id,
ContainerImage = model.ContainerImage.Image,
ContainerTag = model.ContainerImage.Tag, //latestは運用上使用されていないハズなので、そのまま直接代入
DataSetId = model.DataSetId.Value,
EntryPoint = model.EntryPoint,
ModelGitId = gitId.Value,
ModelRepository = model.GitModel.Repository,
ModelRepositoryOwner = model.GitModel.Owner,
ModelBranch = branch,
ModelCommitId = commitId,
OptionDic = model.Options ?? new Dictionary <string, string>(), //オプションはnullの可能性があるので、その時は初期化
Memo = model.Memo,
Cpu = model.Cpu.Value,
Memory = model.Memory.Value,
Gpu = model.Gpu.Value,
Partition = model.Partition,
PortList = model.Ports,
Status = ContainerStatus.Running.Key,
Zip = model.Zip,
LocalDataSet = model.LocalDataSet,
};
if (trainingHistory.OptionDic.ContainsKey("")) //空文字は除外する
{
trainingHistory.OptionDic.Remove("");
}
// 親学習が指定されていれば存在チェック
if (model.ParentIds != null)
{
var maps = new List <TrainingHistoryParentMap>();
foreach (var parentId in model.ParentIds)
{
var parent = await trainingHistoryRepository.GetByIdAsync(parentId);
if (parent == null)
{
return(null, DoJsonNotFound(typeof(TrainingController), requestUrl, modelState, $"Training ID {parentId} is not found."));
}
// 学習履歴に親学習を紐づける
var map = trainingHistoryRepository.AttachParentAsync(trainingHistory, parent);
if (map != null)
{
maps.Add(map);
}
}
trainingHistory.ParentMaps = maps;
}
//タグの登録
if (model.Tags != null && model.Tags.Count() > 0)
{
tagLogic.CreateTrainingHistoryTags(trainingHistory, model.Tags);
}
trainingHistoryRepository.Add(trainingHistory);
if (dataSet.IsLocked == false)
{
dataSet.IsLocked = true;
}
unitOfWork.Commit();
var result = await clusterManagementLogic.RunTrainContainerAsync(trainingHistory, scriptType, regisryTokenName, gitToken);
if (result.IsSuccess == false)
{
//コンテナの起動に失敗した状態。エラーを出力して、保存した学習履歴も削除する。
await dataSetLogic.ReleaseLockAsync(trainingHistory.DataSetId);
trainingHistoryRepository.Delete(trainingHistory);
unitOfWork.Commit();
return(null, DoJsonError(HttpStatusCode.ServiceUnavailable, "Failed to run training. The message bellow may be help to resolve: " + result.Error,
typeof(TrainingController), requestUrl, modelState));
}
//結果に従い、学習結果を更新する。
//実行には時間がかかりうるので、DBから最新の情報を取ってくる
trainingHistory = await trainingHistoryRepository.GetByIdAsync(trainingHistory.Id);
trainingHistory.Configuration = result.Value.Configuration;
trainingHistory.Status = result.Value.Status.Key;
unitOfWork.Commit();
if (result.Value.Status.Succeed())
{
return(trainingHistory,
JsonCreated(new SimpleOutputModel(trainingHistory)));
}
else
{
return(trainingHistory,
DoJsonError(HttpStatusCode.ServiceUnavailable, $"Failed to run training. Status={result.Value.Status.Name}. Please contact your server administrator.",
typeof(TrainingController), requestUrl, modelState));
}
}