/// <summary>
/// ディレクトリ直下のディレクトリ・オブジェクト一覧取得。
/// minio NFSに書き込まれた学習結果を一覧で取得するために使用。
/// 1000件までしか取得しない仕様
/// </summary>
/// <param name="searchDirPath"> 検索対象ディレクトリ </param>
/// <returns>StorageListResultInfo (直下のディレクトリ一覧, 直下のオブジェクト一覧) </returns>
public async Task <Result <StorageListResultInfo, string> > GetUnderDirAsync(string searchDirPath)
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucket,
MaxKeys = 1000,
Prefix = searchDirPath,
Delimiter = "/" // https://dev.classmethod.jp/cloud/aws/amazon-s3-folders/
};
try
{
LogDebug($"start querying objects under bucket");
ListObjectsResponse response = await client.ListObjectsAsync(request);
if (response.IsTruncated)
{
LogWarning("too many output files(should be less than 1000). exceeded files are ignored.");
}
LogDebug($"storeage response : {response.ToString()}");
var dirs = response.CommonPrefixes.Select(x => new StorageDirInfo(x)).ToList();
var files = response.S3Objects.Select(x => new StorageFileInfo(x.Key, x.LastModified, x.Size)).ToList();
var exceeded = response.IsTruncated;
var result = new StorageListResultInfo(dirs, files, exceeded);
return(Result <StorageListResultInfo, string> .CreateResult(result));
}
catch (Exception e)
{
LogDebug($"GetUnderDirAsync error : {e.ToString()}");
return(Result <StorageListResultInfo, string> .CreateErrorResult(e.ToString()));
}
}
/// <summary>
/// バケット直下にディレクトリが存在しなければ、ディレクトリを作成する
/// </summary>
/// <param name="client">使用するS3Client</param>
/// <param name="bucketName">バケット名</param>
private async Task CreateDirUnderBucketAsync(AmazonS3Client client, string bucketName)
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucketName,
MaxKeys = 1000,
Prefix = "",
Delimiter = "/" // https://dev.classmethod.jp/cloud/aws/amazon-s3-folders/
};
try
{
LogDebug($"start querying objects under bucket");
// ディレクトリ・ファイルの一覧を取得
ListObjectsResponse response = await client.ListObjectsAsync(request);
if (response.IsTruncated)
{
LogWarning("too many output files(should be less than 1000). exceeded files are ignored.");
}
LogDebug($"storeage response : {response.ToString()}");
// ディレクトリの存在チェックと作成
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.Data.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingContainerOutputFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingHistoryAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.PreprocContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceHistoryAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceContainerOutputFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.NotebookContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.NotebookContainerOutputFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.ExperimentContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.ExperimentContainerOutputFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.ExperimentPreprocContainerAttachedFiles.ToString());
}
catch (Exception e)
{
LogDebug($"CreateDirUnderBucketAsync error : {e.ToString()}");
}
}