/// <summary>
/// 前処理コンテナを削除し、ステータスを変更する。
/// 削除可否の判断が必要なら、呼び出しもとで判定すること。
/// </summary>
/// <param name="preprocessHistory">対象前処理履歴</param>
/// <param name="force">他テナントに対する変更を許可するか</param>
public async Task <bool> DeleteAsync(PreprocessHistory preprocessHistory, bool force)
{
// 前処理結果を削除
bool result = true;
foreach (var outputDataId in preprocessHistoryRepository.GetPreprocessOutputs(preprocessHistory.Id))
{
//1件でも失敗したら結果はfalse。ただし、エラーが出ても最後まで消し切る。
result &= await dataLogic.DeleteDataAsync(outputDataId);
}
// 前処理履歴の削除
preprocessHistoryRepository.Delete(preprocessHistory, force);
// 結果に関わらずコミット
unitOfWork.Commit();
var status = ContainerStatus.Convert(preprocessHistory.Status);
if (status.Exist())
{
//コンテナが動いていれば、停止する
await clusterManagementLogic.DeleteContainerAsync(
ContainerType.Preprocessing, preprocessHistory.Name, CurrentUserInfo.SelectedTenant.Name, force);
}
return(result);
}
public PreprocessHistoryOutputModel(PreprocessHistory history)
{
Id = history.Id;
InputDataId = history.InputDataId;
InputDataName = history.InputData?.Name;
PreprocessId = history.PreprocessId;
PreprocessName = history.Preprocess?.Name;
}
/// <summary>
/// 前処理履歴作成の入力モデルのチェックを行う。
/// </summary>
/// <param name="preprocessId">前処理ID</param>
/// <param name="inputDataId">入力データID</param>
private async Task <Result <PreprocessHistory, IActionResult> > ValidateCreatePreprocessHistoryInputModelAsync(long preprocessId, long?inputDataId)
{
if (inputDataId == null)
{
return(Result <PreprocessHistory, IActionResult> .CreateErrorResult(JsonBadRequest("Data ID is requried.")));
}
if (!ModelState.IsValid)
{
return(Result <PreprocessHistory, IActionResult> .CreateErrorResult(JsonBadRequest("Invalid Input")));
}
// データIDの存在確認
var data = await dataRepository.GetByIdAsync(inputDataId.Value);
if (data == null)
{
return(Result <PreprocessHistory, IActionResult> .CreateErrorResult(JsonNotFound($"Data ID {inputDataId} is not found.")));
}
// 前処理の存在確認
var preprocess = await preprocessRepository.GetByIdAsync(preprocessId);
if (preprocess == null)
{
return(Result <PreprocessHistory, IActionResult> .CreateErrorResult(JsonNotFound($"Preprocessing ID {preprocessId} is not found.")));
}
// 実行コマンドが指定されていない前処理は起動不能(空白だけでもアウト)
if (string.IsNullOrWhiteSpace(preprocess.EntryPoint))
{
return(Result <PreprocessHistory, IActionResult> .CreateErrorResult(JsonNotFound($"Preprocessing {preprocess.Name} can not be run because of lack of an execution command.")));
}
// イメージが指定されていない前処理は起動不能
if (string.IsNullOrEmpty(preprocess.ContainerImage) || string.IsNullOrEmpty(preprocess.ContainerTag))
{
return(Result <PreprocessHistory, IActionResult> .CreateErrorResult(JsonNotFound($"Preprocessing {preprocess.Name} can not be run because a container image has not been selected properly yet.")));
}
// 前処理が既に実行中か確認する
var preprocessHistory = preprocessHistoryRepository.Find(pph => pph.InputDataId == inputDataId && pph.PreprocessId == preprocess.Id);
if (preprocessHistory != null)
{
string status = ContainerStatus.Convert(preprocessHistory.Status).Name;
return(Result <PreprocessHistory, IActionResult> .CreateErrorResult(JsonNotFound($"Data {data.Id}:{data.Name} has already been processed by {preprocess.Id}:{preprocess.Name}. Status:{status}")));
}
preprocessHistory = new PreprocessHistory()
{
InputDataId = data.Id,
PreprocessId = preprocess.Id,
Preprocess = preprocess,
Status = ContainerStatus.Running.Key
};
return(Result <PreprocessHistory, IActionResult> .CreateResult(preprocessHistory));
}
public async Task <IActionResult> RunPreprocessHistory([FromRoute] long id, [FromBody] RunPreprocessHistoryInputModel model,
[FromServices] IDataRepository dataRepository,
[FromServices] IClusterManagementLogic clusterManagementLogic)
{
var validateResult = await ValidateCreatePreprocessHistoryInputModelAsync(id, model.DataId, dataRepository);
if (validateResult.IsSuccess == false)
{
return(validateResult.Error);
}
PreprocessHistory preprocessHistory = validateResult.Value;
preprocessHistory.Cpu = model.Cpu;
preprocessHistory.Memory = model.Memory;
preprocessHistory.Gpu = model.Gpu;
preprocessHistory.Partition = model.Partition;
preprocessHistory.OptionDic = model.Options ?? new Dictionary <string, string>(); //オプションはnullの可能性があるので、その時は初期化
if (preprocessHistory.OptionDic.ContainsKey("")) //空文字は除外する
{
preprocessHistory.OptionDic.Remove("");
}
preprocessHistoryRepository.Add(preprocessHistory);
unitOfWork.Commit();
//入力データの詳細情報がないので、取得
preprocessHistory.InputData = await dataRepository.GetDataIncludeAllAsync(preprocessHistory.InputDataId);
var result = await clusterManagementLogic.RunPreprocessingContainerAsync(preprocessHistory);
if (result.IsSuccess == false)
{
//コンテナの起動に失敗した状態。エラーを出力して、保存した学習履歴も削除する。
preprocessHistoryRepository.Delete(preprocessHistory);
return(JsonError(HttpStatusCode.ServiceUnavailable, "Failed to run preprocessing. The message bellow may be help to resolve: " + result.Error));
}
//結果に従い、学習結果を更新する。
//実行には時間がかかりうるので、DBから最新の情報を取ってくる
preprocessHistory = await preprocessHistoryRepository.GetByIdAsync(preprocessHistory.Id);
preprocessHistory.Status = result.Value.Status.Key;
unitOfWork.Commit();
if (result.Value.Status.Succeed())
{
return(JsonCreated(new HistoriesOutputModel(preprocessHistory)));
}
else
{
return(JsonError(HttpStatusCode.ServiceUnavailable, $"Failed to run preprocessing. Status={result.Value.Status.Name}. Please contact your server administrator."));
}
}
public HistoriesOutputModel(PreprocessHistory history)
{
Key = history.Name;
Status = ContainerStatus.Convert(history.Status).Name;
CreatedAt = history.CreatedAt.ToFormatedString();
DataId = history.InputDataId;
DataName = history.InputData?.Name;
PreprocessId = history.PreprocessId;
PreprocessName = history.Preprocess?.Name;
}
public async Task <IActionResult> CreatePreprocessHistory([FromRoute] long id, [FromRoute] long?dataId)
{
var result = await ValidateCreatePreprocessHistoryInputModelAsync(id, dataId);
if (result.IsSuccess == false)
{
return(result.Error);
}
PreprocessHistory preprocessHistory = result.Value;
// ステータスはRunningではなくOpenedにする。
preprocessHistory.Status = ContainerStatus.Opened.Key;
preprocessHistoryRepository.Add(preprocessHistory);
unitOfWork.Commit();
return(JsonCreated(new HistoriesOutputModel(preprocessHistory)));
}
/// <summary>
/// ステータスを更新して、出力モデルに変換する
/// </summary>
/// <param name="history">前処理履歴</param>
/// <param name="model">出力モデル</param>
private async Task <HistoriesOutputModel> GetUpdatedIndexOutputModelAsync(PreprocessHistory history, HistoriesOutputModel model)
{
var status = ContainerStatus.Convert(history.Status);
model.StatusType = status.StatusType;
if (status.Exist())
{
// コンテナがまだ存在している場合、情報を更新する
var newStatus = await clusterManagementLogic.GetContainerStatusAsync(history.Name, CurrentUserInfo.SelectedTenant.Name, false);
if (status.Key != newStatus.Key)
{
// 更新があったので、変更処理
history.Status = newStatus.Key;
unitOfWork.Commit();
model.Status = newStatus.Name;
model.StatusType = newStatus.StatusType;
}
}
return(model);
}
public HistoryDetailsOutputModel(PreprocessHistory history) : base(history)
{
}
public async Task <IActionResult> RunPreprocessHistory([FromRoute] long id, [FromBody] RunPreprocessHistoryInputModel model)
{
// 環境変数名のチェック
if (model.Options != null && model.Options.Count > 0)
{
foreach (var env in model.Options)
{
if (!string.IsNullOrEmpty(env.Key))
{
// フォーマットチェック
if (!Regex.IsMatch(env.Key, "^[-._a-zA-Z][-._a-zA-Z0-9]*$"))
{
return(JsonNotFound($"Invalid envName. Please match the format of '^[-._a-zA-Z][-._a-zA-Z0-9]*$'."));
}
}
}
}
// 各リソースの超過チェック
Tenant tenant = tenantRepository.Get(CurrentUserInfo.SelectedTenant.Id);
string errorMessage = clusterManagementLogic.CheckQuota(tenant, model.Cpu.Value, model.Memory.Value, model.Gpu.Value);
if (errorMessage != null)
{
return(JsonBadRequest(errorMessage));
}
var validateResult = await ValidateCreatePreprocessHistoryInputModelAsync(id, model.DataId);
if (validateResult.IsSuccess == false)
{
return(validateResult.Error);
}
PreprocessHistory preprocessHistory = validateResult.Value;
preprocessHistory.Cpu = model.Cpu;
preprocessHistory.Memory = model.Memory;
preprocessHistory.Gpu = model.Gpu;
preprocessHistory.Partition = model.Partition;
preprocessHistory.OptionDic = model.Options ?? new Dictionary <string, string>(); // オプションはnullの可能性があるので、その時は初期化
if (preprocessHistory.OptionDic.ContainsKey("")) // 空文字は除外する
{
preprocessHistory.OptionDic.Remove("");
}
preprocessHistoryRepository.Add(preprocessHistory);
unitOfWork.Commit();
// 入力データの詳細情報がないので、取得
preprocessHistory.InputData = await dataRepository.GetDataIncludeAllAsync(preprocessHistory.InputDataId);
var result = await clusterManagementLogic.RunPreprocessingContainerAsync(preprocessHistory);
if (result.IsSuccess == false)
{
// コンテナの起動に失敗した状態。エラーを出力して、保存した学習履歴も削除する。
preprocessHistoryRepository.Delete(preprocessHistory);
return(JsonError(HttpStatusCode.ServiceUnavailable, "Failed to run preprocessing. The message bellow may be help to resolve: " + result.Error));
}
// 結果に従い、学習結果を更新する。
// 実行には時間がかかりうるので、DBから最新の情報を取ってくる
preprocessHistory = await preprocessHistoryRepository.GetByIdAsync(preprocessHistory.Id);
preprocessHistory.Status = result.Value.Status.Key;
unitOfWork.Commit();
if (result.Value.Status.Succeed())
{
return(JsonCreated(new HistoriesOutputModel(preprocessHistory)));
}
else
{
return(JsonError(HttpStatusCode.ServiceUnavailable, $"Failed to run preprocessing. Status={result.Value.Status.Name}. Please contact your server administrator."));
}
}
/// <summary>
/// 新規に前処理用コンテナを作成する。
/// </summary>
/// <param name="preprocessHistory">対象の前処理履歴</param>
/// <returns>作成したコンテナのステータス</returns>
public async Task <Result <ContainerInfo, string> > RunPreprocessingContainerAsync(PreprocessHistory preprocessHistory)
{
string token = await GetUserAccessTokenAsync();
if (token == null)
{
//トークンがない場合、結果はnull
return(Result <ContainerInfo, string> .CreateErrorResult("Access denied. Failed to get token to access the cluster management system."));
}
var registryMap = registryLogic.GetCurrentRegistryMap(preprocessHistory.Preprocess.ContainerRegistryId.Value);
string tags = "-t " + preprocessHistory.Preprocess.Name; //生成されるデータのタグを設定
foreach (var tag in preprocessHistory.InputData.Tags)
{
tags += " -t " + tag;
}
//コンテナを起動するために必要な設定値をインスタンス化
var inputModel = new RunContainerInputModel()
{
ID = preprocessHistory.Id,
TenantName = TenantName,
LoginUser = CurrentUserInfo.Alias, //アカウントはエイリアスから指定
Name = preprocessHistory.Name,
ContainerImage = registryMap.Registry.GetImagePath(preprocessHistory.Preprocess.ContainerImage, preprocessHistory.Preprocess.ContainerTag),
ScriptType = "preproc", // 実行スクリプトの指定
Cpu = preprocessHistory.Cpu.Value,
Memory = preprocessHistory.Memory.Value,
Gpu = preprocessHistory.Gpu.Value,
KqiToken = loginLogic.GenerateToken().AccessToken,
KqiImage = "kamonohashi/cli:" + versionLogic.GetVersion(),
LogPath = "/kqi/attach/preproc_stdout_stderr_${PREPROCESSING_ID}_${DATA_ID}.log", // 前処理履歴IDは現状ユーザーに見えないので前処理+データIDをつける
NfsVolumeMounts = new List <NfsVolumeMountModel>()
{
// 添付ファイルを保存するディレクトリ
// 前処理結果ディレクトリを前処理完了時にzip圧縮して添付するために使用
new NfsVolumeMountModel()
{
Name = "nfs-preproc-attach",
MountPath = "/kqi/attach",
SubPath = preprocessHistory.Id.ToString(),
Server = CurrentUserInfo.SelectedTenant.Storage.NfsServer,
ServerPath = CurrentUserInfo.SelectedTenant.PreprocContainerAttachedNfsPath,
ReadOnly = false
}
},
ContainerSharedPath = new Dictionary <string, string>()
{
{ "tmp", "/kqi/tmp/" },
{ "input", "/kqi/input/" },
{ "git", "/kqi/git/" },
{ "output", "/kqi/output/" }
},
EnvList = new Dictionary <string, string>()
{
{ "DATA_ID", preprocessHistory.InputDataId.ToString() },
{ "DATA_NAME", preprocessHistory.InputData.Name },
{ "PREPROCESSING_ID", preprocessHistory.PreprocessId.ToString() },
{ "TAGS", tags },
{ "COMMIT_ID", preprocessHistory.Preprocess.RepositoryCommitId },
{ "KQI_SERVER", containerOptions.WebServerUrl },
{ "KQI_TOKEN", loginLogic.GenerateToken().AccessToken },
{ "http_proxy", containerOptions.Proxy },
{ "https_proxy", containerOptions.Proxy },
{ "no_proxy", containerOptions.NoProxy },
{ "HTTP_PROXY", containerOptions.Proxy },
{ "HTTPS_PROXY", containerOptions.Proxy },
{ "NO_PROXY", containerOptions.NoProxy },
{ "COLUMNS", containerOptions.ShellColumns },
{ "PYTHONUNBUFFERED", "true" }, // python実行時の標準出力・エラーのバッファリングをなくす
{ "LC_ALL", "C.UTF-8" }, // python実行時のエラー回避
{ "LANG", "C.UTF-8" } // python実行時のエラー回避
},
EntryPoint = preprocessHistory.Preprocess.EntryPoint,
ClusterManagerToken = token,
RegistryTokenName = registryMap.RegistryTokenKey,
IsNodePort = true
};
// 前処理はGitの未指定も許可するため、その判定
if (preprocessHistory.Preprocess.RepositoryGitId != null)
{
long gitId = preprocessHistory.Preprocess.RepositoryGitId == -1 ?
CurrentUserInfo.SelectedTenant.DefaultGitId.Value : preprocessHistory.Preprocess.RepositoryGitId.Value;
var gitEndpoint = gitLogic.GetPullUrl(preprocessHistory.Preprocess.RepositoryGitId.Value, preprocessHistory.Preprocess.RepositoryName, preprocessHistory.Preprocess.RepositoryOwner);
if (gitEndpoint != null)
{
inputModel.EnvList.Add("MODEL_REPOSITORY", gitEndpoint.FullUrl);
inputModel.EnvList.Add("MODEL_REPOSITORY_URL", gitEndpoint.Url);
inputModel.EnvList.Add("MODEL_REPOSITORY_TOKEN", gitEndpoint.Token);
}
else
{
//Git情報は必須にしているので、無ければエラー
return(Result <ContainerInfo, string> .CreateErrorResult("Git credential is not valid."));
}
}
// ユーザの任意追加環境変数をマージする
AddUserEnvToInputModel(preprocessHistory.OptionDic, inputModel);
//使用できるノードを取得し、制約に追加
inputModel.ConstraintList = new Dictionary <string, List <string> >()
{
{ containerOptions.ContainerLabelHostName, GetAccessibleNode() }
};
if (string.IsNullOrEmpty(preprocessHistory.Partition) == false)
{
// パーティション指定があれば追加
inputModel.ConstraintList.Add(containerOptions.ContainerLabelPartition, new List <string> {
preprocessHistory.Partition
});
}
var outModel = await clusterManagementService.RunContainerAsync(inputModel);
if (outModel.IsSuccess == false)
{
return(Result <ContainerInfo, string> .CreateErrorResult(outModel.Error));
}
return(Result <ContainerInfo, string> .CreateResult(new ContainerInfo()
{
Name = outModel.Value.Name,
Status = outModel.Value.Status,
Host = outModel.Value.Host,
Configuration = outModel.Value.Configuration
}));
}
