public async Task <IActionResult> Delete(long id)
{
var experiment = await experimentRepository
.GetAll()
.Include(x => x.ExperimentPreprocess)
.SingleOrDefaultAsync(x => x.Id == id);
if (experiment == null)
{
return(JsonNotFound($"Experiment ID {id} is not found."));
}
experimentRepository.Delete(experiment);
if (experiment.TrainingHistoryId != null)
{
(var status, var result) = await TrainingController.DoDelete(
experiment.TrainingHistoryId.Value,
trainingHistoryRepository,
clusterManagementLogic,
dataSetLogic,
tagLogic,
unitOfWork,
CurrentUserInfo,
ModelState,
storageLogic,
inferenceHistoryRepository,
tensorBoardContainerRepository,
tagRepository,
RequestUrl);
if (!status)
{
return(result);
}
}
if (experiment.ExperimentPreprocessId != null)
{
var exists = await experimentRepository
.ExistsAsync(x => x.ExperimentPreprocessId == experiment.ExperimentPreprocessId && x.Id != id);
if (!exists)
{
experimentPreprocessRepository.Delete(experiment.ExperimentPreprocess);
(var status, var result) = await TrainingController.DoDelete(
experiment.ExperimentPreprocess.TrainingHistoryId,
trainingHistoryRepository,
clusterManagementLogic,
dataSetLogic,
tagLogic,
unitOfWork,
CurrentUserInfo,
ModelState,
storageLogic,
inferenceHistoryRepository,
tensorBoardContainerRepository,
tagRepository,
RequestUrl);
if (!status)
{
return(result);
}
}
}
unitOfWork.Commit();
return(JsonNoContent());
}
public async Task <IActionResult> CompletePreprocess(long id)
{
var experiment = await experimentRepository
.GetAll()
.Include(x => x.TemplateVersion).ThenInclude(x => x.TrainingContainerRegistry)
.Include(x => x.DataSetVersion)
.Include(x => x.ExperimentPreprocess)
.SingleOrDefaultAsync(x => x.Id == id);
// 実験がない
if (experiment == null)
{
return(JsonNotFound($"Experiment ID {id} is not found."));
}
// 実験に前処理がない
if (experiment.ExperimentPreprocessId == null)
{
return(JsonBadRequest($"Experiment ID {id} does not have preprocess."));
}
// 実験の学習が既に開始している
if (experiment.TrainingHistoryId != null)
{
return(JsonBadRequest($"Experiment ID {id} already has started training."));
}
// kamonohashi学習に必要な情報を設定
var templateVersion = experiment.TemplateVersion;
var dataSetVersion = experiment.DataSetVersion;
var experimentPreprocess = experiment.ExperimentPreprocess;
var registryTokenKey = await RegistRegistryToTenantAsync(templateVersion.TrainingContainerRegistry, templateVersion.TrainingContainerToken);
if (registryTokenKey == null)
{
return(JsonBadRequest("Cannot register registry token"));
}
var gitToken = templateVersion.TrainingRepositoryToken ?? UserGitToken(templateVersion.TrainingRepositoryGitId);
var trainingCreateInputModel = new ApiModels.TrainingApiModels.CreateInputModel
{
Name = experiment.Name,
ContainerImage = new ContainerImageInputModel
{
RegistryId = templateVersion.TrainingContainerRegistryId,
Image = templateVersion.TrainingContainerImage,
Tag = templateVersion.TrainingContainerTag,
},
// 前処理あり実験の学習はデータセットを使用しないが、
// kamonohashi学習ではデータセットを省略できないので、ダミーを指定している。
DataSetId = dataSetVersion.DataSetId,
ParentIds = new List <long> {
experimentPreprocess.TrainingHistoryId,
},
GitModel = new GitCommitInputModel
{
GitId = templateVersion.TrainingRepositoryGitId,
Repository = templateVersion.TrainingRepositoryName,
Owner = templateVersion.TrainingRepositoryOwner,
Branch = templateVersion.TrainingRepositoryBranch,
CommitId = templateVersion.TrainingRepositoryCommitId,
},
EntryPoint = templateVersion.TrainingEntryPoint,
Options = null,
Cpu = templateVersion.TrainingCpu,
Memory = templateVersion.TrainingMemory,
Gpu = templateVersion.TrainingGpu,
Partition = null,
Ports = null,
Memo = $"Training of aquarium. experimentId:{ experiment.Id}",
Tags = null,
Zip = false,
LocalDataSet = false,
};
// kamonohashi学習を開始
(var trainingHistory, var result) = await TrainingController.DoCreate(trainingCreateInputModel,
dataSetRepository, nodeRepository, tenantRepository, trainingHistoryRepository,
clusterManagementLogic, dataSetLogic, gitLogic, tagLogic, unitOfWork,
CurrentUserInfo, ModelState, RequestUrl, "experiment_training_after_preproc", registryTokenKey, gitToken);
// 実験の学習とkamonohashi学習を結び付ける
if (trainingHistory != null)
{
experiment.TrainingHistoryId = trainingHistory.Id;
experimentRepository.Update(experiment);
unitOfWork.Commit();
((JsonResult)result).Value = new SimpleOutputModel(experiment);
}
return(result);
}
/// <summary>
/// 前処理あり実験の前処理を開始する
/// </summary>
private async Task <IActionResult> RunExperimentPreprocess(CreateInputModel model,
TemplateVersion templateVersion, DataSetVersion dataSetVersion)
{
var registryTokenKey = await RegistRegistryToTenantAsync(templateVersion.PreprocessContainerRegistry, templateVersion.PreprocessContainerToken);
if (registryTokenKey == null)
{
return(JsonBadRequest("Cannot register registry token"));
}
var gitToken = templateVersion.PreprocessRepositoryToken ?? UserGitToken(templateVersion.PreprocessRepositoryGitId);
// 実験IDを発行するため、Experimentをここで作成
var experiment = new Experiment
{
Name = model.Name,
DataSetId = model.DataSetId,
DataSetVersionId = model.DataSetVersionId,
TemplateId = model.TemplateId,
TemplateVersionId = model.TemplateVersionId,
};
experimentRepository.Add(experiment);
unitOfWork.Commit();
// kamonohashi学習に必要な情報を設定
var trainingCreateInputModel = new ApiModels.TrainingApiModels.CreateInputModel
{
Name = model.Name,
ContainerImage = new ContainerImageInputModel
{
RegistryId = templateVersion.PreprocessContainerRegistryId,
Image = templateVersion.PreprocessContainerImage,
Tag = templateVersion.PreprocessContainerTag,
},
DataSetId = dataSetVersion.DataSetId,
ParentIds = null,
GitModel = new GitCommitInputModel
{
GitId = templateVersion.PreprocessRepositoryGitId,
Repository = templateVersion.PreprocessRepositoryName,
Owner = templateVersion.PreprocessRepositoryOwner,
Branch = templateVersion.PreprocessRepositoryBranch,
CommitId = templateVersion.PreprocessRepositoryCommitId,
},
EntryPoint = templateVersion.PreprocessEntryPoint,
Options = new Dictionary <string, string>
{
{ "EXPERIMENT_ID", experiment.Id.ToString() },
},
Cpu = templateVersion.PreprocessCpu,
Memory = templateVersion.PreprocessMemory,
Gpu = templateVersion.PreprocessGpu,
Partition = null,
Ports = null,
Memo = $"Preprocess of aquarium. TemplateId:{experiment.TemplateId} ,TemplateVersion:{experiment.TemplateVersion.Version}, DataSetId:{experiment.DataSetId},DataSetVersion:{experiment.DataSetVersion.Version} ",
Tags = null,
Zip = false,
LocalDataSet = false,
};
// kamonohashi学習を開始
(var trainingHistory, var result) = await TrainingController.DoCreate(trainingCreateInputModel,
dataSetRepository, nodeRepository, tenantRepository, trainingHistoryRepository,
clusterManagementLogic, dataSetLogic, gitLogic, tagLogic, unitOfWork,
CurrentUserInfo, ModelState, RequestUrl, "experiment_preproc", registryTokenKey, gitToken);
// 実験の前処理とkamonohashi学習を結び付ける
if (trainingHistory != null)
{
var experimentPreprocess = new ExperimentPreprocess
{
DataSetId = dataSetVersion.AquariumDataSetId,
DataSetVersionId = dataSetVersion.Id,
TemplateId = templateVersion.TemplateId,
TemplateVersionId = templateVersion.Id,
TrainingHistoryId = trainingHistory.Id,
};
experimentPreprocessRepository.Add(experimentPreprocess);
experiment.ExperimentPreprocess = experimentPreprocess;
experimentRepository.Update(experiment);
unitOfWork.Commit();
((JsonResult)result).Value = new SimpleOutputModel(experiment);
}
else
{
experimentRepository.Delete(experiment);
unitOfWork.Commit();
}
return(result);
}
/// <summary>
/// 前処理あり実験の学習を開始する
/// </summary>
private async Task <IActionResult> RunExperimentTrainingWithPreprocess(CreateInputModel model,
TemplateVersion templateVersion, DataSetVersion dataSetVersion, ExperimentPreprocess experimentPreprocess)
{
var registryTokenKey = await RegistRegistryToTenantAsync(templateVersion.TrainingContainerRegistry, templateVersion.TrainingContainerToken);
if (registryTokenKey == null)
{
return(JsonBadRequest("Cannot register registry token"));
}
var gitToken = templateVersion.TrainingRepositoryToken ?? UserGitToken(templateVersion.TrainingRepositoryGitId);
// 実験IDを発行するため、Experimentをここで作成
var experiment = new Experiment
{
Name = model.Name,
DataSetId = model.DataSetId,
DataSetVersionId = model.DataSetVersionId,
TemplateId = model.TemplateId,
TemplateVersionId = model.TemplateVersionId,
};
experimentRepository.Add(experiment);
unitOfWork.Commit();
// kamonohashi学習に必要な情報を設定
var trainingCreateInputModel = new ApiModels.TrainingApiModels.CreateInputModel
{
Name = model.Name,
ContainerImage = new ContainerImageInputModel
{
RegistryId = templateVersion.TrainingContainerRegistryId,
Image = templateVersion.TrainingContainerImage,
Tag = templateVersion.TrainingContainerTag,
},
// 前処理あり実験の学習はデータセットを使用しないが、
// kamonohashi学習ではデータセットを省略できないので、ダミーを指定している。
DataSetId = dataSetVersion.DataSetId,
ParentIds = new List <long> {
experimentPreprocess.TrainingHistoryId,
},
GitModel = new GitCommitInputModel
{
GitId = templateVersion.TrainingRepositoryGitId,
Repository = templateVersion.TrainingRepositoryName,
Owner = templateVersion.TrainingRepositoryOwner,
Branch = templateVersion.TrainingRepositoryBranch,
CommitId = templateVersion.TrainingRepositoryCommitId,
},
EntryPoint = templateVersion.TrainingEntryPoint,
Options = null,
Cpu = templateVersion.TrainingCpu,
Memory = templateVersion.TrainingMemory,
Gpu = templateVersion.TrainingGpu,
Partition = null,
Ports = null,
Memo = $"Training of aquarium. experimentId:{ experiment.Id}",
Tags = null,
Zip = false,
LocalDataSet = false,
};
// kamonohashi学習を開始
(var trainingHistory, var result) = await TrainingController.DoCreate(trainingCreateInputModel,
dataSetRepository, nodeRepository, tenantRepository, trainingHistoryRepository,
clusterManagementLogic, dataSetLogic, gitLogic, tagLogic, unitOfWork,
CurrentUserInfo, ModelState, RequestUrl, "experiment_training_after_preproc", registryTokenKey, gitToken);
// 実験の学習とkamonohashi学習を結び付ける
if (trainingHistory != null)
{
experiment.ExperimentPreprocessId = experimentPreprocess.Id;
experiment.TrainingHistoryId = trainingHistory.Id;
experimentRepository.Update(experiment);
unitOfWork.Commit();
((JsonResult)result).Value = new SimpleOutputModel(experiment);
}
else
{
experimentRepository.Delete(experiment);
unitOfWork.Commit();
}
return(result);
}
/// <summary>
/// kamonohashi学習のステータスを更新し、最新ステータスを返す
/// </summary>
private async Task <ContainerStatus> UpdateStatus(TrainingHistory history)
=> history != null
? await TrainingController.DoGetUpdatedIndexOutputModelAsync(history,
clusterManagementLogic, CurrentUserInfo, trainingHistoryRepository, unitOfWork)
: ContainerStatus.None;
