public static void AsyncListObjects()
{
try
{
var listObjectsRequest = new ListObjectsRequest(bucketName);
ossClient.BeginListObjects(listObjectsRequest, ListObjectCallback, null);
evnet.WaitOne();
}
catch (OssException ex)
{
Console.WriteLine("Failed with error code: {0}; Error info: {1}. \nRequestID:{2}\tHostID:{3}",
ex.ErrorCode, ex.Message, ex.RequestId, ex.HostId);
}
catch (Exception ex)
{
Console.WriteLine("Failed with error info: {0}", ex.Message);
}
}
public List <string> ListObjectsInBucket(string folderKey)
{
List <string> keys = new List <string>();
try
{
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = _bucketName;
request.Prefix = folderKey;
ListObjectsResponse response = _awsS3Client.ListObjects(request);
foreach (S3Object o in response.S3Objects)
{
keys.Add(o.Key);
}
}
catch (Exception ex)
{
throw new ApplicationException("S3 ListObjectsInBucket ", ex);
}
return(keys);
}
IList <string> ICoreAmazonS3.GetAllObjectKeys(string bucketName, string prefix, IDictionary <string, object> additionalProperties)
{
var keys = new List <string>();
string marker = null;
do
{
var request = new ListObjectsRequest
{
BucketName = bucketName,
Prefix = prefix,
Marker = marker
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
var listResponse = this.ListObjects(request);
keys.AddRange(listResponse.S3Objects.Select(o => o.Key));
marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(marker));
return(keys);
}
/// <summary>
/// To use this and other API operations, you must be authorized in an IAM policy. If you are not authorized, talk to an administrator.
/// If you are an administrator who needs to write policies to give users access, see Getting Started with Policies.
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public async Task <ListObjectsResponse> ListObjects(ListObjectsRequest request)
{
var uri = new Uri($"{GetEndPointNoneVersion(ObjectStorageServices.Object(request.NamespaceName, request.BucketName), this.Region)}?{request.GetOptionQuery()}");
var webResponse = await this.RestClientAsync.Get(uri, new HttpRequestHeaderParam()
{
OpcClientRequestId = request.OpcClientRequestId
});
using (var stream = webResponse.GetResponseStream())
using (var reader = new StreamReader(stream))
{
var response = reader.ReadToEnd();
return(new ListObjectsResponse()
{
ListObjects = JsonSerializer.Deserialize <ListObjectDetails>(response),
OpcRequestId = webResponse.Headers.Get("opc-request-id"),
OpcClientRequestId = webResponse.Headers.Get("opc-client-request-id")
});
}
}
/// <summary>
/// バケット直下にディレクトリが存在しなければ、ディレクトリを作成する
/// </summary>
/// <param name="client">使用するS3Client</param>
/// <param name="bucketName">バケット名</param>
private async Task CreateDirUnderBucketAsync(AmazonS3Client client, string bucketName)
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucketName,
MaxKeys = 1000,
Prefix = "",
Delimiter = "/" // https://dev.classmethod.jp/cloud/aws/amazon-s3-folders/
};
try
{
LogDebug($"start querying objects under bucket");
// ディレクトリ・ファイルの一覧を取得
ListObjectsResponse response = await client.ListObjectsAsync(request);
if (response.IsTruncated)
{
LogWarning("too many output files(should be less than 1000). exceeded files are ignored.");
}
LogDebug($"storeage response : {response.ToString()}");
// ディレクトリの存在チェックと作成
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.Data.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingContainerOutputFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingHistoryAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.PreprocContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceHistoryAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceContainerOutputFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.NotebookContainerAttachedFiles.ToString());
await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.NotebookContainerOutputFiles.ToString());
}
catch (Exception e)
{
LogDebug($"CreateDirUnderBucketAsync error : {e.ToString()}");
}
}
public virtual IEnumerable <string> GetDirectories(string path)
{
if (string.IsNullOrEmpty(path))
{
path = "/";
}
path = ResolveBucketPath(path, true);
var request = new ListObjectsRequest
{
BucketName = BucketName,
Delimiter = Delimiter,
Prefix = path
};
var response = ExecuteWithContinuation(request);
return(response
.SelectMany(p => p.CommonPrefixes)
.Select(p => RemovePrefix(p))
.ToArray());
}
/// <summary>
/// Download new/Updated AmazonS3 files and return AmazonS3 keys
/// </summary>
/// <returns></returns>
public List <String> FileDownloader()
{
List <String> ret = new List <String>();
try
{
using (AmazonS3Client s3Client = GetAmazonS3Client()) {
try
{
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = AmazonS3Config.GetConfigData.AmazonS3BucketName;
Task <ListObjectsResponse> task = s3Client.ListObjectsAsync(request);
task.Wait();
ListObjectsResponse response = task.Result;
List <AmazonS3ConfigDataKeys> keys = AmazonS3Config.GetConfigData.Keys;
foreach (S3Object item in response.S3Objects)
{
AmazonS3ConfigDataKeys itemF = keys.FirstOrDefault(o => o.AmazonS3Key == item.Key);
try
{
if (itemF != null)
{
ReadAmazonFile(s3Client, item, itemF);
}
ret.Add(item.Key);
}
catch (Exception) { }
}
} catch (Exception)
{
}
}
}
catch (Exception) { }
return(ret);
}
public IEnumerable <S3Object> GetFiles(String prefix = null)
{
var objects = new List <S3Object>();
String lastKey = "";
String preLastKey = "";
do
{
preLastKey = lastKey;
ListObjectsRequest request;
if (String.IsNullOrWhiteSpace(prefix))
{
request = new ListObjectsRequest();
request.BucketName = _bucket;
}
else
{
request = new ListObjectsRequest();
request.BucketName = _bucket;
request.Prefix = prefix;
}
request.Marker = lastKey;
ListObjectsResponse response = _client.ListObjects(request);
var objectListing = response.S3Objects;
foreach (S3Object obj in objectListing)
{
// Exclude directories.
Boolean isDirectory = obj.Key.EndsWith("/");
if (!isDirectory)
{
objects.Add(obj);
lastKey = obj.Key;
}
}
} while (lastKey != preLastKey);
return(objects);
}
private IEnumerable <S3Object> GetS3Objects(string domain, string path)
{
using (AmazonS3 client = GetClient())
{
var request = new ListObjectsRequest
{
BucketName = _bucket,
Prefix = (MakePath(domain, path.TrimEnd('/') + '/')),
MaxKeys = (1000)
};
var objects = new List <S3Object>();
ListObjectsResponse response;
do
{
response = client.ListObjects(request);
objects.AddRange(response.S3Objects.Where(entry => CheckKey(domain, entry.Key)));
request.Marker = response.NextMarker;
} while (response.IsTruncated);
return(objects);
}
}
public List <string> GetFileUrls(string bucketName, string folder)
{
var fileUrls = new List <string>();
try
{
folder = folder.TrimEnd('\\', '/') + "/";
var client = new OssClient(_endPoint, _accessKeyId, _accessKeySecret);
ObjectListing result;
var nextMarker = string.Empty;
do
{
// 每页列举的文件个数通过maxKeys指定，超过指定数将进行分页显示。
var listObjectsRequest = new ListObjectsRequest(bucketName)
{
Prefix = folder,
Marker = nextMarker,
MaxKeys = 100
};
result = client.ListObjects(listObjectsRequest);
foreach (var summary in result.ObjectSummaries)
{
var url = summary.Key.Remove(0, folder.Length);
if (!string.IsNullOrEmpty(url))
{
fileUrls.Add($"https://{bucketName}.{_endPoint.TrimEnd('\\', '/')}/{summary.Key}");
}
}
nextMarker = result.NextMarker;
} while (result.IsTruncated);
}
catch (Exception e)
{
throw new Exception(e.Message);
}
return(fileUrls);
}
public string GetBucketFileListHtml()
{
StringBuilder output = new StringBuilder();
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = BucketName;
ListObjectsResponse listResponse;
do
{
// Get a list of objects
listResponse = client.ListObjects(request);
foreach (S3Object obj in listResponse.S3Objects)
{
output.AppendFormat("<li>{0}</li>", obj.Key);
}
// Set the marker property
request.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated);
return(output.ToString());
}
public static List <S3Object> ListObjects(string bucketName, string prefix = null, int?maxEntries = null)
{
using (IAmazonS3 s3Client = GetAmazonS3ClientInstance())
{
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = bucketName;
if (!string.IsNullOrEmpty(prefix))
{
request.Prefix = prefix;
}
if (maxEntries.HasValue)
{
request.MaxKeys = maxEntries.Value;
}
ListObjectsResponse response = s3Client.ListObjects(request);
return(response.S3Objects);
}
}
//private static readonly OssClient client = new OssClient(Config.Endpoint, Config.AccessKeyId, Config.AccessKeySecret);
#endregion
#region 步获取object列表
/// <summary>
/// 同步获取object列表
/// </summary>
/// <param name="bucketName"></param>
/// <param name="prefix">搜索前缀</param>
/// <returns>object名称列表</returns>
public List <string> ListObjectsWithRequest(string bucketName, string prefix)
{
try
{
var keys = new List <string>();
ObjectListing result;
string nextMarker = string.Empty;
do
{
var listObjectsRequest = new ListObjectsRequest(bucketName)
{
Marker = nextMarker, Prefix = prefix, MaxKeys = 100
};
result = _client.ListObjects(listObjectsRequest);
foreach (var summary in result.ObjectSummaries)
{
Console.WriteLine(summary.Key);
keys.Add(summary.Key);
}
nextMarker = result.NextMarker;
} while (result.IsTruncated);
Console.WriteLine("List objects of bucket:{0} succeeded ", bucketName);
return(keys);
}
catch (OssException ex)
{
Console.WriteLine("Failed with error code: {0}; Error info: {1}. \nRequestID:{2}\tHostID:{3}",
ex.ErrorCode, ex.Message, ex.RequestId, ex.HostId);
}
catch (Exception ex)
{
Console.WriteLine("Failed with error info: {0}", ex.Message);
}
return(null);
}
public static async Task BatchDeleteObjects(string bucketName)
{
try
{
var listObjectsRequest = new ListObjectsRequest()
{
BucketName = bucketName,
Prefix = "folder/sub_folder"
};
var result = await Sample.Client.ListObjectsAsync(listObjectsRequest);
var response = await Sample.Client.DeleteObjectsAsync(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = result.S3Objects
.Select(o => new KeyVersion
{
Key = o.Key
}).ToList(),
Quiet = false
});
response.DeletedObjects.ForEach(o =>
{
Console.WriteLine("Batch Delete object:{0}", o.Key);
});
Console.WriteLine("Batch Delete objects succeeded");
}
catch (AmazonServiceException ex)
{
Console.WriteLine("Failed with error code: {0}; Error info: {1}. \nRequestID:{2}",
ex.ErrorCode, ex.Message, ex.RequestId);
}
catch (Exception ex)
{
Console.WriteLine("Failed with error info: {0}", ex.Message);
}
}
public async Task TraverseFor(Settings settings)
{
var request = new ListObjectsRequest()
{
BucketName = settings.BucketName,
Prefix = settings.SourcePrefix
};
var objects = new List <Amazon.S3.Model.S3Object>();
var response = await _s3Client.ListObjectsAsync(request);
objects.AddRange(response.S3Objects);
while (response.IsTruncated)
{
request.Marker = response.NextMarker;
response = await _s3Client.ListObjectsAsync(request);
objects.AddRange(response.S3Objects);
}
foreach (var s3Object in objects)
{
var extension = s3Object.Key.Substring(s3Object.Key.LastIndexOf(".") + 1);
if (!_acceptedExtensions.Any(x => x.Equals(extension, StringComparison.InvariantCultureIgnoreCase)))
{
continue;
}
var recognitionRequest = new RecognitionRequest()
{
Age = settings.Age,
IsSmiling = settings.IsSmiling,
IsMale = settings.IsMale
};
var result = await _faceRecogniser.Recognise(settings.BucketName, s3Object.Key, recognitionRequest);
if (result.IsFound)
{
await _objectUploader.UploadObjectAsync(settings.BucketName, s3Object.Key, $"{settings.DestinationPrefix}/{s3Object.Key.GetFileFromKey()}");
}
}
}
/// <summary>
/// Query S3 Objects
/// </summary>
/// <param name="bucketName">Bucket name</param>
/// <param name="prefix">Limits the response to keys that begin with the specified prefix</param>
/// <param name="marker">Specifies the key to start with when listing objects in a bucket</param>
/// <param name="maxKeys"> Sets the maximum number of keys returned in the response. The response might contain fewer keys but will never contain more.</param>
/// <returns>IEnumerable<TCloudBlobObject></returns>
public IEnumerable <S3Object> QueryS3Objects(string bucketName, string prefix = null,
string marker = null, int maxKeys = 100)
{
try
{
bucketName.CheckNullObject(nameof(bucketName));
var request = new ListObjectsRequest
{
BucketName = bucketName,
MaxKeys = maxKeys,
Prefix = prefix,
Marker = marker
};
return(blobClient.ListObjects(request).S3Objects);
}
catch (Exception ex)
{
throw ex.Handle(new { bucketName, prefix, marker, maxKeys });
}
}
private IEnumerable <S3Object> GetS3Objects(string domain, string path)
{
using (AmazonS3 client = GetClient())
{
var request = new ListObjectsRequest {
BucketName = _bucket
};
request.WithPrefix(MakePath(domain, path.TrimEnd('/') + '/')).WithMaxKeys(1000);
var objects = new List <S3Object>();
ListObjectsResponse response;
do
{
response = client.ListObjects(request);
objects.AddRange(response.S3Objects.Where(entry => CheckKey(domain, entry.Key)));
if (objects.Count > 0)
{
request.Marker = objects[objects.Count - 1].Key;
}
} while (response.IsTruncated);
return(objects);
}
}
/// <summary>
/// Copies an InputStream to Amazon S3 and grants access to MessageGears.
/// </summary>
/// <param name="stream">
/// An input stream for the data to be copied to S3.
/// </param>
/// <param name="bucketName">
/// The name of the S3 bucket where the file will be copied.
/// </param>
/// <param name="key">
/// The S3 key of the file to be created.
/// </param>
public void PutS3File(Stream stream, String bucketName, String key)
{
// Check to see if the file already exists in S3
ListObjectsRequest listRequest = new ListObjectsRequest().WithBucketName(bucketName).WithPrefix(key);
ListObjectsResponse listResponse = listFiles(listRequest);
if (listResponse.S3Objects.Count > 0)
{
String message = "File " + key + " already exists.";
log.Warn("PutS3File failed: " + message);
throw new ApplicationException(message);
}
// Copy a file to S3
PutObjectRequest request = new PutObjectRequest().WithKey(key).WithBucketName(bucketName).WithTimeout(properties.S3PutTimeoutSecs * 1000);
request.WithInputStream(stream);
putWithRetry(request);
setS3PermissionsWithRetry(bucketName, key);
log.Info("PutS3File successful: " + key);
}
/// <summary>
/// Gets the attachment count.
/// </summary>
/// <param name="path">The path.</param>
/// <returns></returns>
/// <exception cref="System.NotImplementedException"></exception>
public int GetFileCount(string path, string thumbnailPath)
{
_logger.LogTrace("Entering GetFileCount in AmazonS3UploadService");
try
{
var regionEndPoint = RegionEndpoint.GetBySystemName(_awsParams.Region);
IAmazonS3 client = new AmazonS3Client(_awsParams.AccessKeyId, _awsParams.SecretKey, regionEndPoint);
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = _awsParams.BucketName,
Prefix = path.TrimEnd('/') + "/",
};
var response = client.ListObjectsAsync(request).GetAwaiter().GetResult();
var count = response.S3Objects.Select(x => x.Key).Count();
return(count);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
throw new InvalidOperationException("GetFileCount failed in AmazonS3UploadService");
}
}
public async Task <Blob> Bind(string fileKey)
{
using (var client = GetClient())
{
var listRequest = new ListObjectsRequest
{
BucketName = TempBucket,
Prefix = fileKey.WithSuffix("/"),
};
var response = await client.ListObjectsAsync(listRequest);
if (response.S3Objects.Count == 0)
{
throw new Exception("There is no file in the temp folder " + fileKey);
}
if (response.S3Objects.Count > 1)
{
throw new Exception("There are multiple files in the temp folder " + fileKey);
}
var obj = response.S3Objects.Single();
var objectRequest = new GetObjectRequest
{
BucketName = TempBucket,
Key = obj.Key,
};
var file = await client.GetObjectAsync(objectRequest);
var content = await file.ResponseStream.ReadAllBytesAsync();
var filename = file.Key.RemoveBeforeAndIncluding(fileKey.WithSuffix("/"));
return(new Blob(content, filename));
}
}
/// <summary>
/// 列出指定容器下的对象列表
/// </summary>
/// <param name="containerName"></param>
/// <returns></returns>
public async Task <IList <BlobFileInfo> > ListBlobs(string containerName)
{
var blobFileInfos = new List <BlobFileInfo>();
try
{
return(await Task.Run(() =>
{
if (!string.IsNullOrWhiteSpace(containerName) && !containerName.EndsWith("/"))
{
containerName += "/";
}
var listObjectsRequest = new ListObjectsRequest(_cfg.BucketName)
{
Prefix = containerName
};
var result = _ossClient.ListObjects(listObjectsRequest).HandlerError("获取对象列表出错！");
foreach (var summary in result.ObjectSummaries)
{
blobFileInfos.Add(new BlobFileInfo
{
Container = summary.BucketName,
ETag = summary.ETag,
LastModified = summary.LastModified,
Name = summary.Key,
Length = summary.Size,
Url = string.Format(_baseUrl, summary.BucketName, summary.Key)
});
}
return blobFileInfos;
}));
}
catch (Exception ex)
{
throw new StorageException(StorageErrorCode.PostError.ToStorageError(),
new Exception(ex.ToString()));
}
}
/// <summary>
/// Diagnostics each object returned by S3
/// </summary>
private bool ScanBucket()
{
Logger.WriteLog(ErrorCodes.SyncPhase_SendingBegun3,
ErrorResources.SyncPhase_SendingBegun, Severity.Information, VerboseLevel.User, true);
Logger.WriteLog(ErrorCodes.S3BucketSource_GettingObjectList,
ErrorResources.S3BucketSource_GettingObjectList, Severity.Information, VerboseLevel.User);
// Gets the summary about an object
ListObjectsRequest request = new ListObjectsRequest()
{
BucketName = _bucketName,
MaxKeys = int.MaxValue
};
ListObjectsResponse list = _client.ListObjectsAsync(request).Result;
for (int i = 0; i < list.S3Objects.Count; i++)
{
ScanProgress(i, list.S3Objects.Count, list.S3Objects[i].Key);
// Gets the meta data about an object
var meta = _client.GetObjectMetadataAsync(new GetObjectMetadataRequest()
{
BucketName = _bucketName,
Key = list.S3Objects[i].Key
}).Result.Metadata;
if (!ProcessObject(list.S3Objects[i], meta))
{
break;
}
}
Logger.WriteLog(ErrorCodes.SyncPhase_SendingEnded3,
ErrorResources.SyncPhase_SendingEnded, Severity.Information, VerboseLevel.User, true);
return(true);
}
public void DeleteFolder(string folderPath)
{
try
{
folderPath = CorrectFolderPath(folderPath);
// get entire contents of source folder
ListObjectsRequest listRequest = new ListObjectsRequest()
{
BucketName = _bucketName,
Prefix = folderPath
};
ListObjectsResponse listResponse = _awsS3Client.ListObjects(listRequest);
List <KeyVersion> objectKeys = new List <KeyVersion>();
foreach (var s3Object in listResponse.S3Objects)
{
objectKeys.Add(new KeyVersion {
Key = s3Object.Key
});
}
// empty folders do not really exist in S3 so don't try to delete an empty folder
if (objectKeys.Count > 0)
{
DeleteObjectsRequest request = new DeleteObjectsRequest
{
Objects = objectKeys,
BucketName = _bucketName
};
DeleteObjectsResponse response = _awsS3Client.DeleteObjects(request);
}
}
catch (Exception ex)
{
throw new ApplicationException("S3 DeleteFolder - unexpected exception.", ex);
}
}
// Delete bucket functionality
public static void DeleteBucketFunctionality(string NameOfTheBucket, string RegionOfTheBucket)
{
RegionOfTheBucket = RegionOfTheBucket.ToLower();
try
{
AmazonS3Client client = new AmazonS3Client(AccessKey, SecretKey, Amazon.RegionEndpoint.GetBySystemName(ClientRegion(RegionOfTheBucket)));
ListObjectsRequest ObjectRequest = new ListObjectsRequest
{
BucketName = NameOfTheBucket
};
ListObjectsResponse ListResponse;
do
{
ListResponse = client.ListObjects(ObjectRequest);
foreach (S3Object obj in ListResponse.S3Objects)
{
DeleteObjectRequest DeleteObject = new DeleteObjectRequest
{
BucketName = NameOfTheBucket,
Key = obj.Key
};
client.DeleteObject(DeleteObject);
}
ObjectRequest.Marker = ListResponse.NextMarker;
} while (ListResponse.IsTruncated);
DeleteBucketRequest DeleteRequest = new DeleteBucketRequest
{
BucketName = NameOfTheBucket,
UseClientRegion = true
};
client.DeleteBucket(DeleteRequest);
Console.WriteLine("Bucket Deleted");
}
catch (Exception e)
{
Console.WriteLine("ERROR MESSAGE : " + e.Message);
}
}
private async Task <IList <string> > GetKeysForPrefix(string prefix)
{
var keys = new List <string>();
var request = new ListObjectsRequest
{
BucketName = _options.BucketName,
Prefix = prefix
};
using (var client = CreateS3Client())
{
ListObjectsResponse response = await client.ListObjectsAsync(request);
foreach (S3Object obj in response.S3Objects)
{
keys.Add(obj.Key);
}
}
return(keys);
}
public void size(string bucket)
{
int count = 0;
long filesize = 0;
string nextmarker = "";
try
{
while (nextmarker != null)
{
ListObjectsRequest listrequest = new ListObjectsRequest();
listrequest.BucketName = bucket;
listrequest.Marker = nextmarker;
ListObjectsResponse listresponse = s3Client.ListObjects(listrequest);
foreach (S3Object s3obj in listresponse.S3Objects)
{
count++;
filesize += s3obj.Size;
}
if (listresponse.S3Objects.Count == 1000)
{
nextmarker = listresponse.S3Objects[999].Key;
}
else
{
nextmarker = null;
}
}
Console.WriteLine("Bucket: " + bucket);
Console.WriteLine("File Count: " + count.ToString());
Console.WriteLine("Total File Size: " + filesize.ToString());
}
catch (Exception e)
{
// report error and retry
Console.WriteLine("Error retrieving bucket list for bucket '" + bucket + "': " + e.Message);
}
}
/// <summary>
/// Gets a list of object Keys when given the specified parameters. Please note that the maximum value of takeXRecords is determined by the bucket setup.
/// </summary>
/// <param name="currentConnectionProfile">Contains the profile information driving the type of connection we will use with Amazon S3.</param>
/// <param name="takeXRecords">Specifies how many records to pull. This might be limited to 1000 records, as set up in the Amazon account.</param>
/// <returns>A list of entity S3MediaEntity, where the Key is probably the most important data used.</returns>
public List <S3MediaEntity> GetS3MediaList(S3ConnectionProfileEntity currentConnectionProfile, int takeXRecords)
{
try
{
var s3Client = GetAmazonS3Client();
var listObjectsRequest = new ListObjectsRequest
{
MaxKeys = takeXRecords,
BucketName = currentConnectionProfile.BucketName,
Prefix = currentConnectionProfile.FolderName
};
var files = s3Client.ListObjects(listObjectsRequest);
var mediaList = new List <S3MediaEntity>();
foreach (var record in files.S3Objects)
{
if (record.Key.Contains("_$folder$"))
{
continue;
}
var media = new S3MediaEntity()
{
ProfileName = currentConnectionProfile.ProfileName,
BucketName = currentConnectionProfile.BucketName,
FolderName = currentConnectionProfile.FolderName,
Key = record.Key
};
mediaList.Add(media);
}
return(mediaList);
}
catch (Exception exception)
{
throw new SystemException(exception.Message);
}
}
private async Task <List <Secret> > GetSecrets(string filter, AmazonS3Client s3Client)
{
var secrets = new List <Secret>();
var listRequest = new ListObjectsRequest {
BucketName = BucketName
};
if (!string.IsNullOrEmpty(filter))
{
listRequest.Prefix = filter;
}
ListObjectsResponse response;
do
{
logger.Info($"list secrets from marker {listRequest.Marker}");
response = await s3Client.ListObjectsAsync(listRequest);
foreach (var summary in response.S3Objects)
{
var objectKey = summary.Key;
if (!objectKey.EndsWith(".isec"))
{
continue;
}
var secret = new Secret(objectKey.Substring(0, objectKey.LastIndexOf(".isec")));
secret.LastModified = summary.LastModified;
secrets.Add(secret);
}
listRequest.Marker = response.NextMarker;
} while (response.IsTruncated);
return(secrets);
}
public void FindCaseInAWSAsync(int caseNumber)
{
var request = new ListObjectsRequest()
{
BucketName = bucketValue
};
bool caseFound = false;
S3Client.ListObjectsAsync(request, (responseObject) =>
{
if (responseObject.Exception == null)
{
if (responseObject.Response.S3Objects.Count > 0)
{
responseObject.Response.S3Objects.ForEach((o) =>
{
int responseCaseNumber = int.Parse(o.Key.Substring(4, o.Key.IndexOf('.') - 4));
if (caseNumber == responseCaseNumber)
{
downloadCase(caseNumber);
Debug.Log("Found Case# " + caseNumber);
caseFound = true;
return;
}
});
}
}
else
{
Debug.Log("****** AWS get exception! *****");
}
if (!caseFound)
{
Debug.Log("****** NO SUCH CASE EXISTS! ******");
}
});
}
/// <summary>
/// 列表对象
/// </summary>
/// <param name="bucketName"></param>
/// <param name="prefix"></param>
/// <returns></returns>
public Task <List <Item> > ListObjectsAsync(string bucketName, string prefix = null)
{
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException(nameof(bucketName));
}
List <Item> result = new List <Item>();
ListObjectsResponse resultObj = null;
string nextMarker = string.Empty;
do
{
// 每页列举的文件个数通过maxKeys指定，超过指定数将进行分页显示。
var listObjectsRequest = new ListObjectsRequest()
{
BucketName = bucketName,
Prefix = prefix,
Marker = nextMarker,
MaxKeys = 100
};
resultObj = _client.ListObjects(listObjectsRequest);
if (resultObj == null)
{
continue;
}
foreach (var item in resultObj.ObsObjects)
{
result.Add(new Item()
{
Key = item.ObjectKey,
LastModified = item.LastModified.ToString(),
ETag = item.ETag,
Size = (ulong)item.Size,
BucketName = bucketName,
IsDir = !string.IsNullOrWhiteSpace(item.ObjectKey) && item.ObjectKey[^ 1] == '/',
LastModifiedDateTime = item.LastModified,
});
}
public override bool Execute()
{
RequireSecretKey();
AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(AccessKey, SecretKey);
WarnIfUneven(Tuple.Create("Buckets", Buckets), Tuple.Create("Folders", Folders));
foreach (var itemPair in Zip(Buckets, Folders))
{
string bucketName = itemPair.Item1.ItemSpec;
string outputFolder = itemPair.Item2.ItemSpec;
string filePath;
string directory;
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = bucketName;
RequireDirectory(outputFolder);
Log.LogMessage(MessageImportance.High, "Listing Bucket \"{0}\"", bucketName);
using (ListObjectsResponse response = s3Client.ListObjects(request))
{
foreach (S3Object entry in response.S3Objects)
{
filePath = Path.Combine(outputFolder, entry.Key.Replace('/', '\\'));
directory = Path.GetDirectoryName(filePath);
Log.LogMessage(MessageImportance.Normal, "Listed \"{0}\"", filePath);
if (!filePath.EndsWith("\\"))
{
RequireDirectory(directory);
DateTimeOffset lastModified = DateTimeOffset.ParseExact(entry.LastModified, CultureInfo.InvariantCulture.DateTimeFormat.RFC1123Pattern, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
File.WriteAllText(filePath, entry.LastModified + " -> " + lastModified.LocalDateTime);
File.SetLastWriteTime(filePath, lastModified.LocalDateTime);
}
}
}
}
return(true);
}
public static void ListObjectsWithRequest(string bucketName)
{
try
{
var keys = new List<string>();
ObjectListing result = null;
string nextMarker = string.Empty;
do
{
var listObjectsRequest = new ListObjectsRequest(bucketName)
{
Marker = nextMarker,
MaxKeys = 100
};
result = client.ListObjects(listObjectsRequest);
foreach (var summary in result.ObjectSummaries)
{
Console.WriteLine(summary.Key);
keys.Add(summary.Key);
}
nextMarker = result.NextMarker;
} while (result.IsTruncated);
Console.WriteLine("List objects of bucket:{0} succeeded ", bucketName);
}
catch (OssException ex)
{
Console.WriteLine("Failed with error code: {0}; Error info: {1}. \nRequestID:{2}\tHostID:{3}",
ex.ErrorCode, ex.Message, ex.RequestId, ex.HostId);
}
catch (Exception ex)
{
Console.WriteLine("Failed with error info: {0}", ex.Message);
}
}
public List<string> ListAll()
{
List<string> files = new List<string>();
using ( AmazonS3 client = AWSClientFactory.CreateAmazonS3Client( this.amazonKey, this.amazonSecret ) ) {
ListObjectsRequest request = new ListObjectsRequest {
BucketName = this.amazonBucket
};
do {
ListObjectsResponse response = client.ListObjects( request );
foreach ( S3Object s3Object in response.S3Objects ) {
files.Add( s3Object.Key );
}
// If response is truncated, get the next page
if ( response.IsTruncated ) {
request.Marker = response.NextMarker;
} else {
request = null;
}
} while ( request != null );
}
return files;
}
/// <summary>
/// Queries a bucket for a listing of objects it contains. Only objects with keys
/// beginning with the given prefix will be returned. The DefaultDelimiter will
/// be used. If you expect a large number of objects to be returned, consider using
/// ListAllObjects().
/// </summary>
public IList<ListEntry> ListObjects(string bucketName, string prefix)
{
var args = new ListObjectsArgs { Prefix = prefix, Delimiter = DefaultDelimiter };
var request = new ListObjectsRequest(this, bucketName, args);
using (ListObjectsResponse response = request.GetResponse())
{
if (response.IsTruncated)
throw new Exception("The server truncated the list of items requested. Consider using the ListObjectsRequest class to query for large numbers of items.");
return response.Entries.ToList();
}
}
/// <summary>
/// Queries a bucket for a listing of objects it contains. Only objects with keys
/// beginning with the given prefix will be returned.
/// </summary>
public IEnumerable<ListEntry> ListAllObjects(string bucketName, string prefix, string delimiter)
{
var args = new ListObjectsArgs
{
Prefix = prefix,
Delimiter = delimiter
};
while (true)
{
var request = new ListObjectsRequest(this, bucketName, args);
using (var response = request.GetResponse())
{
ListEntry lastEntry = null;
foreach (var entry in response.Entries)
{
lastEntry = entry;
yield return entry;
}
if (response.IsTruncated)
{
// if you specified a delimiter, S3 is supposed to give us the marker
// name to use in order to get the next set of "stuff".
if (response.NextMarker != null)
args.Marker = response.NextMarker;
// if you didn't specify a delimiter, you won't get any CommonPrefixes,
// so we'll use the last ObjectEntry's key as the next delimiter.
else if (lastEntry is ObjectEntry)
args.Marker = (lastEntry as ObjectEntry).Key;
else
throw new Exception("S3 Server is misbehaving.");
}
else
break; // we're done!
}
}
}
private static void DeleteObjectsOfBucket(string bucketName)
{
var keys = new List<string>();
ObjectListing result = null;
string nextMarker = string.Empty;
do
{
var listObjectsRequest = new ListObjectsRequest(bucketName)
{
Marker = nextMarker,
MaxKeys = 100
};
result = client.ListObjects(listObjectsRequest);
foreach (var summary in result.ObjectSummaries)
{
keys.Add(summary.Key);
}
nextMarker = result.NextMarker;
if (keys.Count != 0)
{
client.DeleteObjects(new DeleteObjectsRequest(bucketName, keys));
keys.Clear();
}
} while (result.IsTruncated);
}
public ListObjectsResponse ListObjects(ListObjectsRequest request)
{
EnsureAbsoluteUri(request);
var command = new ListObjects(_userId, _secret, _builder, _authenticator) { Parameters = request };
return (ListObjectsResponse)((ICommandExecutor) this).Execute(command);
}
