/// <summary>
/// コンストラクタ
/// </summary>
public InferenceController(
ITrainingHistoryRepository trainingHistoryRepository,
IInferenceHistoryRepository inferenceHistoryRepository,
IDataSetRepository dataSetRepository,
ITenantRepository tenantRepository,
INodeRepository nodeRepository,
IDataSetLogic dataSetLogic,
IInferenceLogic inferenceLogic,
IStorageLogic storageLogic,
IGitLogic gitLogic,
IClusterManagementLogic clusterManagementLogic,
IUnitOfWork unitOfWork,
IHttpContextAccessor accessor) : base(accessor)
{
this.trainingHistoryRepository = trainingHistoryRepository;
this.inferenceHistoryRepository = inferenceHistoryRepository;
this.dataSetRepository = dataSetRepository;
this.tenantRepository = tenantRepository;
this.nodeRepository = nodeRepository;
this.dataSetLogic = dataSetLogic;
this.inferenceLogic = inferenceLogic;
this.storageLogic = storageLogic;
this.gitLogic = gitLogic;
this.clusterManagementLogic = clusterManagementLogic;
this.unitOfWork = unitOfWork;
}
/// <summary>
/// コンストラクタ
/// </summary>
public TrainingController(
ITrainingHistoryRepository trainingHistoryRepository,
IInferenceHistoryRepository inferenceHistoryRepository,
ITensorBoardContainerRepository tensorBoardContainerRepository,
IDataSetRepository dataSetRepository,
ITagRepository tagRepository,
ITenantRepository tenantRepository,
INodeRepository nodeRepository,
IDataSetLogic dataSetLogic,
ITagLogic tagLogic,
ITrainingLogic trainingLogic,
IStorageLogic storageLogic,
IGitLogic gitLogic,
IClusterManagementLogic clusterManagementLogic,
IOptions <ContainerManageOptions> containerOptions,
IUnitOfWork unitOfWork,
IHttpContextAccessor accessor) : base(accessor)
{
this.trainingHistoryRepository = trainingHistoryRepository;
this.inferenceHistoryRepository = inferenceHistoryRepository;
this.tensorBoardContainerRepository = tensorBoardContainerRepository;
this.dataSetRepository = dataSetRepository;
this.tagRepository = tagRepository;
this.tenantRepository = tenantRepository;
this.nodeRepository = nodeRepository;
this.dataSetLogic = dataSetLogic;
this.tagLogic = tagLogic;
this.trainingLogic = trainingLogic;
this.storageLogic = storageLogic;
this.gitLogic = gitLogic;
this.clusterManagementLogic = clusterManagementLogic;
this.containerOptions = containerOptions.Value;
this.unitOfWork = unitOfWork;
}
public ExperimentController(
IExperimentRepository experimentRepository,
IExperimentPreprocessRepository experimentPreprocessRepository,
IAquariumDataSetRepository aquariumDataSetRepository,
IDataSetRepository dataSetRepository,
IGitRepository gitRepository,
IRegistryRepository registryRepository,
ITrainingHistoryRepository trainingHistoryRepository,
ITemplateRepository templateRepository,
ITemplateVersionRepository templateVersionRepository,
ITenantRepository tenantRepository,
INodeRepository nodeRepository,
ITemplateLogic templateLogic,
ITagLogic tagLogic,
IGitLogic gitLogic,
IClusterManagementLogic clusterManagementLogic,
IDataSetLogic dataSetLogic,
IRegistryLogic registryLogic,
IUnitOfWork unitOfWork,
IInferenceHistoryRepository inferenceHistoryRepository,
ITensorBoardContainerRepository tensorBoardContainerRepository,
ITagRepository tagRepository,
IStorageLogic storageLogic,
IHttpContextAccessor accessor) : base(accessor)
{
this.trainingHistoryRepository = trainingHistoryRepository;
this.experimentRepository = experimentRepository;
this.experimentPreprocessRepository = experimentPreprocessRepository;
this.aquariumDataSetRepository = aquariumDataSetRepository;
this.dataSetRepository = dataSetRepository;
this.gitRepository = gitRepository;
this.registryRepository = registryRepository;
this.templateRepository = templateRepository;
this.templateVersionRepository = templateVersionRepository;
this.tenantRepository = tenantRepository;
this.nodeRepository = nodeRepository;
this.templateLogic = templateLogic;
this.tagLogic = tagLogic;
this.gitLogic = gitLogic;
this.clusterManagementLogic = clusterManagementLogic;
this.dataSetLogic = dataSetLogic;
this.registryLogic = registryLogic;
this.inferenceHistoryRepository = inferenceHistoryRepository;
this.tensorBoardContainerRepository = tensorBoardContainerRepository;
this.tagRepository = tagRepository;
this.storageLogic = storageLogic;
this.unitOfWork = unitOfWork;
}
public AquariumDataSetController(
IAquariumDataSetRepository aquariumDataSetRepository,
IAquariumDataSetVersionRepository aquariumDataSetVersionRepository,
IExperimentRepository experimentRepository,
IExperimentPreprocessRepository experimentPreprocessRepository,
IDataSetRepository dataSetRepository,
IDataTypeRepository dataTypeRepository,
IDataSetLogic dataSetLogic,
IUnitOfWork unitOfWork,
IHttpContextAccessor accessor) : base(accessor)
{
this.aquariumDataSetRepository = aquariumDataSetRepository;
this.aquariumDataSetVersionRepository = aquariumDataSetVersionRepository;
this.experimentRepository = experimentRepository;
this.experimentPreprocessRepository = experimentPreprocessRepository;
this.dataSetRepository = dataSetRepository;
this.dataTypeRepository = dataTypeRepository;
this.dataSetLogic = dataSetLogic;
this.unitOfWork = unitOfWork;
}
public static async Task <(TrainingHistory, IActionResult)> DoCreate(CreateInputModel model,
IDataSetRepository dataSetRepository,
INodeRepository nodeRepository,
ITenantRepository tenantRepository,
ITrainingHistoryRepository trainingHistoryRepository,
IClusterManagementLogic clusterManagementLogic,
IDataSetLogic dataSetLogic,
IGitLogic gitLogic,
ITagLogic tagLogic,
IUnitOfWork unitOfWork,
UserInfo currentUserInfo,
ModelStateDictionary modelState,
string requestUrl,
string scriptType,
string regisryTokenName,
string gitToken
)
{
//データの入力チェック
if (!modelState.IsValid)
{
return(null,
DoJsonBadRequest(typeof(TrainingController), requestUrl, modelState, "Invalid inputs."));
}
//データの存在チェック
var dataSet = await dataSetRepository.GetByIdAsync(model.DataSetId.Value);
if (dataSet == null)
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"DataSet ID {model.DataSetId} is not found."));
}
if (string.IsNullOrEmpty(model.Partition) == false)
{
bool existPartition = await nodeRepository.IsEnablePartitionAsync(model.Partition, true);
if (existPartition == false)
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"There are no enable nodes with Partition {model.Partition}."));
}
}
// 環境変数名のチェック
if (model.Options != null && model.Options.Count > 0)
{
foreach (var env in model.Options)
{
if (!string.IsNullOrEmpty(env.Key))
{
// フォーマットチェック
if (!Regex.IsMatch(env.Key, "^[-._a-zA-Z][-._a-zA-Z0-9]*$"))
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState, $"Invalid envName. Please match the format of '^[-._a-zA-Z][-._a-zA-Z0-9]*$'."));
}
}
}
}
long? gitId = model.GitModel.GitId ?? currentUserInfo.SelectedTenant.DefaultGit?.Id;
string branch = model.GitModel.Branch ?? "master";
string commitId = model.GitModel.CommitId;
//コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(commitId))
{
commitId = await gitLogic.GetCommitIdAsync(gitId.Value, model.GitModel.Repository, model.GitModel.Owner, branch);
if (string.IsNullOrEmpty(commitId))
{
//コミットIDが特定できなかったらエラー
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"The branch {branch} for {gitId.Value}/{model.GitModel.Owner}/{model.GitModel.Repository} is not found."));
}
}
// 各リソースの超過チェック
Tenant tenant = tenantRepository.Get(currentUserInfo.SelectedTenant.Id);
string errorMessage = clusterManagementLogic.CheckQuota(tenant, model.Cpu.Value, model.Memory.Value, model.Gpu.Value);
if (errorMessage != null)
{
return(null, DoJsonBadRequest(typeof(TrainingController), requestUrl, modelState, errorMessage));
}
//コンテナの実行前に、学習履歴を作成する（コンテナの実行に失敗した場合、そのステータスをユーザに表示するため）
var trainingHistory = new TrainingHistory()
{
Name = model.Name,
DisplayId = -1,
ContainerRegistryId = model.ContainerImage.RegistryId ?? currentUserInfo.SelectedTenant.DefaultRegistry?.Id,
ContainerImage = model.ContainerImage.Image,
ContainerTag = model.ContainerImage.Tag, //latestは運用上使用されていないハズなので、そのまま直接代入
DataSetId = model.DataSetId.Value,
EntryPoint = model.EntryPoint,
ModelGitId = gitId.Value,
ModelRepository = model.GitModel.Repository,
ModelRepositoryOwner = model.GitModel.Owner,
ModelBranch = branch,
ModelCommitId = commitId,
OptionDic = model.Options ?? new Dictionary <string, string>(), //オプションはnullの可能性があるので、その時は初期化
Memo = model.Memo,
Cpu = model.Cpu.Value,
Memory = model.Memory.Value,
Gpu = model.Gpu.Value,
Partition = model.Partition,
PortList = model.Ports,
Status = ContainerStatus.Running.Key,
Zip = model.Zip,
LocalDataSet = model.LocalDataSet,
};
if (trainingHistory.OptionDic.ContainsKey("")) //空文字は除外する
{
trainingHistory.OptionDic.Remove("");
}
// 親学習が指定されていれば存在チェック
if (model.ParentIds != null)
{
var maps = new List <TrainingHistoryParentMap>();
foreach (var parentId in model.ParentIds)
{
var parent = await trainingHistoryRepository.GetByIdAsync(parentId);
if (parent == null)
{
return(null, DoJsonNotFound(typeof(TrainingController), requestUrl, modelState, $"Training ID {parentId} is not found."));
}
// 学習履歴に親学習を紐づける
var map = trainingHistoryRepository.AttachParentAsync(trainingHistory, parent);
if (map != null)
{
maps.Add(map);
}
}
trainingHistory.ParentMaps = maps;
}
//タグの登録
if (model.Tags != null && model.Tags.Count() > 0)
{
tagLogic.CreateTrainingHistoryTags(trainingHistory, model.Tags);
}
trainingHistoryRepository.Add(trainingHistory);
if (dataSet.IsLocked == false)
{
dataSet.IsLocked = true;
}
unitOfWork.Commit();
var result = await clusterManagementLogic.RunTrainContainerAsync(trainingHistory, scriptType, regisryTokenName, gitToken);
if (result.IsSuccess == false)
{
//コンテナの起動に失敗した状態。エラーを出力して、保存した学習履歴も削除する。
await dataSetLogic.ReleaseLockAsync(trainingHistory.DataSetId);
trainingHistoryRepository.Delete(trainingHistory);
unitOfWork.Commit();
return(null, DoJsonError(HttpStatusCode.ServiceUnavailable, "Failed to run training. The message bellow may be help to resolve: " + result.Error,
typeof(TrainingController), requestUrl, modelState));
}
//結果に従い、学習結果を更新する。
//実行には時間がかかりうるので、DBから最新の情報を取ってくる
trainingHistory = await trainingHistoryRepository.GetByIdAsync(trainingHistory.Id);
trainingHistory.Configuration = result.Value.Configuration;
trainingHistory.Status = result.Value.Status.Key;
unitOfWork.Commit();
if (result.Value.Status.Succeed())
{
return(trainingHistory,
JsonCreated(new SimpleOutputModel(trainingHistory)));
}
else
{
return(trainingHistory,
DoJsonError(HttpStatusCode.ServiceUnavailable, $"Failed to run training. Status={result.Value.Status.Name}. Please contact your server administrator.",
typeof(TrainingController), requestUrl, modelState));
}
}