/// <summary>
/// コンストラクタ
/// </summary>
public TrainingController(
ITrainingHistoryRepository trainingHistoryRepository,
IInferenceHistoryRepository inferenceHistoryRepository,
ITensorBoardContainerRepository tensorBoardContainerRepository,
IDataSetRepository dataSetRepository,
ITagRepository tagRepository,
ITenantRepository tenantRepository,
INodeRepository nodeRepository,
ITagLogic tagLogic,
ITrainingLogic trainingLogic,
IStorageLogic storageLogic,
IGitLogic gitLogic,
IClusterManagementLogic clusterManagementLogic,
IOptions <ContainerManageOptions> containerOptions,
IUnitOfWork unitOfWork,
IHttpContextAccessor accessor) : base(accessor)
{
this.trainingHistoryRepository = trainingHistoryRepository;
this.inferenceHistoryRepository = inferenceHistoryRepository;
this.tensorBoardContainerRepository = tensorBoardContainerRepository;
this.dataSetRepository = dataSetRepository;
this.tagRepository = tagRepository;
this.tenantRepository = tenantRepository;
this.nodeRepository = nodeRepository;
this.tagLogic = tagLogic;
this.trainingLogic = trainingLogic;
this.storageLogic = storageLogic;
this.gitLogic = gitLogic;
this.clusterManagementLogic = clusterManagementLogic;
this.containerOptions = containerOptions.Value;
this.unitOfWork = unitOfWork;
}
/// <summary>
/// コンストラクタ
/// </summary>
public DataLogic(
IDataRepository dataRepository,
ITagLogic tagLogic,
IStorageLogic storageLogic,
ICommonDiLogic commonDiLogic) : base(commonDiLogic)
{
this.dataRepository = dataRepository;
this.tagLogic = tagLogic;
this.storageLogic = storageLogic;
}
public ExperimentController(
IExperimentRepository experimentRepository,
IExperimentPreprocessRepository experimentPreprocessRepository,
IAquariumDataSetRepository aquariumDataSetRepository,
IDataSetRepository dataSetRepository,
IGitRepository gitRepository,
IRegistryRepository registryRepository,
ITrainingHistoryRepository trainingHistoryRepository,
ITemplateRepository templateRepository,
ITemplateVersionRepository templateVersionRepository,
ITenantRepository tenantRepository,
INodeRepository nodeRepository,
ITemplateLogic templateLogic,
ITagLogic tagLogic,
IGitLogic gitLogic,
IClusterManagementLogic clusterManagementLogic,
IDataSetLogic dataSetLogic,
IRegistryLogic registryLogic,
IUnitOfWork unitOfWork,
IInferenceHistoryRepository inferenceHistoryRepository,
ITensorBoardContainerRepository tensorBoardContainerRepository,
ITagRepository tagRepository,
IStorageLogic storageLogic,
IHttpContextAccessor accessor) : base(accessor)
{
this.trainingHistoryRepository = trainingHistoryRepository;
this.experimentRepository = experimentRepository;
this.experimentPreprocessRepository = experimentPreprocessRepository;
this.aquariumDataSetRepository = aquariumDataSetRepository;
this.dataSetRepository = dataSetRepository;
this.gitRepository = gitRepository;
this.registryRepository = registryRepository;
this.templateRepository = templateRepository;
this.templateVersionRepository = templateVersionRepository;
this.tenantRepository = tenantRepository;
this.nodeRepository = nodeRepository;
this.templateLogic = templateLogic;
this.tagLogic = tagLogic;
this.gitLogic = gitLogic;
this.clusterManagementLogic = clusterManagementLogic;
this.dataSetLogic = dataSetLogic;
this.registryLogic = registryLogic;
this.inferenceHistoryRepository = inferenceHistoryRepository;
this.tensorBoardContainerRepository = tensorBoardContainerRepository;
this.tagRepository = tagRepository;
this.storageLogic = storageLogic;
this.unitOfWork = unitOfWork;
}
public DataController(
IDataRepository dataRepository,
IPreprocessHistoryRepository preprocessHistoryRepository,
IDataLogic dataLogic,
ITagLogic tagLogic,
IStorageLogic storageLogic,
IUnitOfWork unitOfWork,
IHttpContextAccessor accessor) : base(accessor)
{
this.dataRepository = dataRepository;
this.preprocessHistoryRepository = preprocessHistoryRepository;
this.dataLogic = dataLogic;
this.tagLogic = tagLogic;
this.storageLogic = storageLogic;
this.unitOfWork = unitOfWork;
}
/// <summary>
/// コンストラクタ
/// </summary>
public PreprocessingController(
IPreprocessRepository preprocessRepository,
IPreprocessHistoryRepository preprocessHistoryRepository,
ITenantRepository tenantRepository,
IDataRepository dataRepository,
IPreprocessLogic preprocessLogic,
ITagLogic tagLogic,
IGitLogic gitLogic,
IStorageLogic storageLogic,
IClusterManagementLogic clusterManagementLogic,
IUnitOfWork unitOfWork,
IHttpContextAccessor accessor) : base(accessor)
{
this.preprocessRepository = preprocessRepository;
this.preprocessHistoryRepository = preprocessHistoryRepository;
this.tenantRepository = tenantRepository;
this.dataRepository = dataRepository;
this.preprocessLogic = preprocessLogic;
this.tagLogic = tagLogic;
this.gitLogic = gitLogic;
this.storageLogic = storageLogic;
this.clusterManagementLogic = clusterManagementLogic;
this.unitOfWork = unitOfWork;
}
public TagController(ITagLogic tagLogic)
{
_tagLogic = tagLogic;
}
public async Task <IActionResult> UploadPreprocessImage([FromRoute] long id, [FromRoute] long dataId,
[FromBody] AddOutputDataInputModel model,
[FromServices] ITagLogic tagLogic,
[FromServices] IDataRepository dataRepository)
{
if (!ModelState.IsValid)
{
return(JsonBadRequest("Invalid inputs."));
}
//データの存在チェック
var preprocessHistory = await preprocessHistoryRepository.GetPreprocessIncludeDataAndPreprocessAsync(id, dataId);
if (preprocessHistory == null)
{
return(JsonNotFound($"Preprocessing History about Preprocess {id} to Data {dataId} is not found."));
}
var status = ContainerStatus.Convert(preprocessHistory.Status);
if (status.IsOpened() == false)
{
//追加できるのは開放中のコンテナだけ（ローカルの結果を追加することがあるので、Runningとは限らない）
return(JsonBadRequest($"Preprocessing History {preprocessHistory.Id} is not opened."));
}
//データを追加する
Data newData = new Data()
{
//データ名が未指定であれば、デフォルトの値を入れる
Name = string.IsNullOrEmpty(model.Name) ? $"{preprocessHistory.InputData.Name}_{preprocessHistory.Preprocess.Name}" : model.Name,
Memo = model.Memo,
ParentDataId = preprocessHistory.InputDataId
};
dataRepository.Add(newData);
foreach (var file in model.Files)
{
dataRepository.AddFile(newData, file.FileName, file.StoredPath);
}
//タグの登録
if (model.Tags != null && model.Tags.Count() > 0)
{
tagLogic.Create(newData, model.Tags);
}
else
{
//タグが未指定であれば、前処理名を付ける
List <string> tags = new List <string>()
{
preprocessHistory.Preprocess.Name
};
tagLogic.Create(newData, tags);
}
preprocessHistoryRepository.AddOutputData(preprocessHistory.Id, newData);
unitOfWork.Commit();
return(JsonOK(new HistoriesOutputModel(preprocessHistory)));
}
public static async Task <(TrainingHistory, IActionResult)> DoCreate(CreateInputModel model,
IDataSetRepository dataSetRepository,
INodeRepository nodeRepository,
ITenantRepository tenantRepository,
ITrainingHistoryRepository trainingHistoryRepository,
IClusterManagementLogic clusterManagementLogic,
IDataSetLogic dataSetLogic,
IGitLogic gitLogic,
ITagLogic tagLogic,
IUnitOfWork unitOfWork,
UserInfo currentUserInfo,
ModelStateDictionary modelState,
string requestUrl,
string scriptType,
string regisryTokenName,
string gitToken
)
{
//データの入力チェック
if (!modelState.IsValid)
{
return(null,
DoJsonBadRequest(typeof(TrainingController), requestUrl, modelState, "Invalid inputs."));
}
//データの存在チェック
var dataSet = await dataSetRepository.GetByIdAsync(model.DataSetId.Value);
if (dataSet == null)
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"DataSet ID {model.DataSetId} is not found."));
}
if (string.IsNullOrEmpty(model.Partition) == false)
{
bool existPartition = await nodeRepository.IsEnablePartitionAsync(model.Partition, true);
if (existPartition == false)
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"There are no enable nodes with Partition {model.Partition}."));
}
}
// 環境変数名のチェック
if (model.Options != null && model.Options.Count > 0)
{
foreach (var env in model.Options)
{
if (!string.IsNullOrEmpty(env.Key))
{
// フォーマットチェック
if (!Regex.IsMatch(env.Key, "^[-._a-zA-Z][-._a-zA-Z0-9]*$"))
{
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState, $"Invalid envName. Please match the format of '^[-._a-zA-Z][-._a-zA-Z0-9]*$'."));
}
}
}
}
long? gitId = model.GitModel.GitId ?? currentUserInfo.SelectedTenant.DefaultGit?.Id;
string branch = model.GitModel.Branch ?? "master";
string commitId = model.GitModel.CommitId;
//コミットIDが指定されていなければ、ブランチのHEADからコミットIDを取得する
if (string.IsNullOrEmpty(commitId))
{
commitId = await gitLogic.GetCommitIdAsync(gitId.Value, model.GitModel.Repository, model.GitModel.Owner, branch);
if (string.IsNullOrEmpty(commitId))
{
//コミットIDが特定できなかったらエラー
return(null,
DoJsonNotFound(typeof(TrainingController), requestUrl, modelState,
$"The branch {branch} for {gitId.Value}/{model.GitModel.Owner}/{model.GitModel.Repository} is not found."));
}
}
// 各リソースの超過チェック
Tenant tenant = tenantRepository.Get(currentUserInfo.SelectedTenant.Id);
string errorMessage = clusterManagementLogic.CheckQuota(tenant, model.Cpu.Value, model.Memory.Value, model.Gpu.Value);
if (errorMessage != null)
{
return(null, DoJsonBadRequest(typeof(TrainingController), requestUrl, modelState, errorMessage));
}
//コンテナの実行前に、学習履歴を作成する（コンテナの実行に失敗した場合、そのステータスをユーザに表示するため）
var trainingHistory = new TrainingHistory()
{
Name = model.Name,
DisplayId = -1,
ContainerRegistryId = model.ContainerImage.RegistryId ?? currentUserInfo.SelectedTenant.DefaultRegistry?.Id,
ContainerImage = model.ContainerImage.Image,
ContainerTag = model.ContainerImage.Tag, //latestは運用上使用されていないハズなので、そのまま直接代入
DataSetId = model.DataSetId.Value,
EntryPoint = model.EntryPoint,
ModelGitId = gitId.Value,
ModelRepository = model.GitModel.Repository,
ModelRepositoryOwner = model.GitModel.Owner,
ModelBranch = branch,
ModelCommitId = commitId,
OptionDic = model.Options ?? new Dictionary <string, string>(), //オプションはnullの可能性があるので、その時は初期化
Memo = model.Memo,
Cpu = model.Cpu.Value,
Memory = model.Memory.Value,
Gpu = model.Gpu.Value,
Partition = model.Partition,
PortList = model.Ports,
Status = ContainerStatus.Running.Key,
Zip = model.Zip,
LocalDataSet = model.LocalDataSet,
};
if (trainingHistory.OptionDic.ContainsKey("")) //空文字は除外する
{
trainingHistory.OptionDic.Remove("");
}
// 親学習が指定されていれば存在チェック
if (model.ParentIds != null)
{
var maps = new List <TrainingHistoryParentMap>();
foreach (var parentId in model.ParentIds)
{
var parent = await trainingHistoryRepository.GetByIdAsync(parentId);
if (parent == null)
{
return(null, DoJsonNotFound(typeof(TrainingController), requestUrl, modelState, $"Training ID {parentId} is not found."));
}
// 学習履歴に親学習を紐づける
var map = trainingHistoryRepository.AttachParentAsync(trainingHistory, parent);
if (map != null)
{
maps.Add(map);
}
}
trainingHistory.ParentMaps = maps;
}
//タグの登録
if (model.Tags != null && model.Tags.Count() > 0)
{
tagLogic.CreateTrainingHistoryTags(trainingHistory, model.Tags);
}
trainingHistoryRepository.Add(trainingHistory);
if (dataSet.IsLocked == false)
{
dataSet.IsLocked = true;
}
unitOfWork.Commit();
var result = await clusterManagementLogic.RunTrainContainerAsync(trainingHistory, scriptType, regisryTokenName, gitToken);
if (result.IsSuccess == false)
{
//コンテナの起動に失敗した状態。エラーを出力して、保存した学習履歴も削除する。
await dataSetLogic.ReleaseLockAsync(trainingHistory.DataSetId);
trainingHistoryRepository.Delete(trainingHistory);
unitOfWork.Commit();
return(null, DoJsonError(HttpStatusCode.ServiceUnavailable, "Failed to run training. The message bellow may be help to resolve: " + result.Error,
typeof(TrainingController), requestUrl, modelState));
}
//結果に従い、学習結果を更新する。
//実行には時間がかかりうるので、DBから最新の情報を取ってくる
trainingHistory = await trainingHistoryRepository.GetByIdAsync(trainingHistory.Id);
trainingHistory.Configuration = result.Value.Configuration;
trainingHistory.Status = result.Value.Status.Key;
unitOfWork.Commit();
if (result.Value.Status.Succeed())
{
return(trainingHistory,
JsonCreated(new SimpleOutputModel(trainingHistory)));
}
else
{
return(trainingHistory,
DoJsonError(HttpStatusCode.ServiceUnavailable, $"Failed to run training. Status={result.Value.Status.Name}. Please contact your server administrator.",
typeof(TrainingController), requestUrl, modelState));
}
}
