public virtual IEnumerable <IFileEntry> ListBucket(string bucketName, string prefix) { bool isTruncated = true; string filename = null; //TODO: Figure out if this is the case with AWSSDK too //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results //So, track the files that have already been returned and skip any duplicates. HashSet <string> alreadyReturned = new HashSet <string>(); //We truncate after ITEM_LIST_LIMIT elements, and then repeat while (isTruncated) { ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.BucketName = bucketName; if (!string.IsNullOrEmpty(filename)) { listRequest.Marker = filename; } listRequest.MaxKeys = ITEM_LIST_LIMIT; if (!string.IsNullOrEmpty(prefix)) { listRequest.Prefix = prefix; } ListObjectsResponse listResponse = m_client.ListObjects(listRequest); isTruncated = listResponse.IsTruncated; filename = listResponse.NextMarker; foreach (S3Object obj in listResponse.S3Objects) { if (alreadyReturned.Add(obj.Key)) { yield return(new Common.IO.FileEntry( obj.Key, obj.Size, obj.LastModified, obj.LastModified )); } } } }
// Delete folder // Returns true on success, false on error. public override bool DeleteFolder(String bucket) { try { this.Exception = null; // First delete any objects in the bucket List <String> objects = new List <string>(); ListObjectsRequest request = new ListObjectsRequest() { BucketName = bucket }; ListObjectsResponse response = this.StorageClient.ListObjects(request); if (response != null && response.S3Objects != null) { foreach (S3Object obj in response.S3Objects) { objects.Add(obj.Key); } } foreach (String key in objects) { this.StorageClient.DeleteObject(bucket, key); } // Now delete the empty bucket this.StorageClient.DeleteBucket(bucket); return(true); } catch (Exception ex) { this.Exception = ex; if (!this.HandleErrors) { throw ex; } return(false); } }
public async Task ListObjectsWithEncoding() { await CreateTempBucketAsync(async bucket => { string tempObjName = "!#/()"; await ObjectClient.PutObjectStringAsync(bucket, tempObjName, string.Empty).ConfigureAwait(false); ListObjectsResponse resp = await ObjectClient.ListObjectsAsync(bucket, req => req.EncodingType = EncodingType.Url).ConfigureAwait(false); Assert.True(resp.IsSuccess); Assert.Equal(EncodingType.Url, resp.EncodingType); S3Object obj = Assert.Single(resp.Objects); Assert.Equal("%21%23/%28%29", obj.ObjectKey); }).ConfigureAwait(false); }
public override long GetFileSize(string domain, string path) { using (AmazonS3 client = GetClient()) { var request = new ListObjectsRequest { BucketName = _bucket }; request.Prefix = (MakePath(domain, path)); using (ListObjectsResponse response = client.ListObjects(request)) { if (response.S3Objects.Count > 0) { return(response.S3Objects[0].Size); } throw new FileNotFoundException("file not found", path); } } }
public ICommandResponse ParseResponse(WebResponse webResponse) { var listResponse = new ListObjectsResponse { Policy = webResponse.Headers["x-emc-policy"], Token = webResponse.Headers["x-emc-token"] ?? string.Empty }; var responseStream = webResponse.GetResponseStream(); if ((responseStream != null) && (responseStream.CanRead)) { var reader = new StreamReader(responseStream); listResponse.Content = XDocument.Load(reader); } return(listResponse); }
private long QuotaDelete(string domain, AmazonS3 client, string key) { if (QuotaController != null) { using ( ListObjectsResponse responce = client.ListObjects(new ListObjectsRequest().WithBucketName(_bucket).WithPrefix(key))) { if (responce.S3Objects != null && responce.S3Objects.Count > 0) { long size = Convert.ToInt64(responce.S3Objects[0].Size); QuotaController.QuotaUsedDelete(_modulename, domain, _dataList.GetData(domain), size); return(size); } } } return(0); }
void ListObjectsCallback(AmazonServiceResult result) { runningResult = null; if (result.Exception == null) { ListObjectsResponse response = result.Response as ListObjectsResponse; foreach (S3Object ob in response.S3Objects) { runningResult += "Find object: " + ob.Key + "\n"; } } else { Debug.LogException(result.Exception); Debug.LogError("ListObject fail"); runningResult = "ListObject fail: " + result.Exception.Message; } }
public List <ImageFile> GetItems(string bucket) { List <ImageFile> imageFiles = new List <ImageFile>(); ListObjectsRequest request = new ListObjectsRequest(); request.BucketName = bucket; ListObjectsResponse response = client.ListObjects(request); foreach (S3Object o in response.S3Objects.Where(m => m.Key.EndsWith(".jpg"))) { ImageFile imageFile = new ImageFile(); imageFile.Pixels = new Collection <Pixel>(); imageFile.FileName = o.Key; imageFile.FileSize = o.Size; imageFiles.Add(imageFile); } return(imageFiles); }
/// <summary> /// Lists the folders within a storage provider's path. /// </summary> /// <param name="path">The relative path to the folder which folders to list.</param> /// <returns>The list of folders in the folder.</returns> public IEnumerable <IStorageFolder> ListFolders(string path) { EnsureInitialized(); if (_client == null) { Logger.Warning(""); return(null); } var folders = new List <S3StorageFolder>(); List <S3Object> s3Objects = new List <S3Object>(); path = path ?? ""; ListObjectsRequest request = new ListObjectsRequest(); request.BucketName = BucketName; request.Prefix = path; request.MaxKeys = 5000; var depth = path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries).Length; ListObjectsResponse response = _client.ListObjects(request); s3Objects.AddRange(response.S3Objects); while (response.IsTruncated) { request.Marker = response.NextMarker; response = _client.ListObjects(request); s3Objects.AddRange(response.S3Objects); } var folderObjects = s3Objects .Where(o => o.Key.EndsWith("/") && o.Key.StartsWith(path) && o.Key.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries).Length == depth + 1) .ToList(); foreach (var fo in folderObjects) { var folderSize = ListFiles(fo.Key).Sum(x => x.GetSize()); folders.Add(new S3StorageFolder(fo, folderSize)); } return(folders); }
public IEnumerable <SimpleFile> GetSimulationInputFiles() { // S3:URL in description string bucketName = "Simulations"; string key = job.SimulationId.ToString(); //string dest = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "name.bin"); IAWSContext awsCtx = Turbine.Consumer.AWS.AppUtility.GetContext(); byte[] bytes; using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(awsCtx.AccessKey, awsCtx.SecretKey)) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .WithBucketName(bucketName) .WithDelimiter("/") .WithPrefix(String.Format("/{0}/StagedInputFiles/", key)); using (ListObjectsResponse listObjectsResponse = client.ListObjects(listObjectsRequest)) { foreach (S3Object obj in listObjectsResponse.S3Objects) { GetObjectRequest getObjectRequest = new GetObjectRequest() .WithBucketName(bucketName) .WithKey(String.Format("/{0}/StagedInputFiles/{1}", key, obj.Key)); using (S3Response getObjectResponse = client.GetObject(getObjectRequest)) { using (System.IO.Stream s = getObjectResponse.ResponseStream) { using (var ms = new System.IO.MemoryStream()) { s.CopyTo(ms); bytes = ms.ToArray(); } } } var f = new SimpleFile() { content = bytes, name = obj.Key }; yield return(f); } } } }
/// <summary> /// Gaets list of items from a bucket. /// </summary> public IList <S3Object> List(string bucketName) { try { ListObjectsRequest listObjectsRequest = new ListObjectsRequest { BucketName = bucketName }; ListObjectsResponse response = TransferUtility.S3Client.ListObjects(listObjectsRequest); return(response.S3Objects); } catch (AmazonS3Exception e) { LogAmazonS3Exception("Getting the list of objects has failed.", e); throw; } }
private static async Task GetAllObjectsAsync() { ListBucketsResponse listBucketResponse = await localclient.ListBucketsAsync(); foreach (S3Bucket bucket in listBucketResponse.Buckets) { Console.Out.WriteLine("Bucket '" + bucket.BucketName + "' created at " + bucket.CreationDate); } if (listBucketResponse.Buckets.Count > 0) { string bucketName = listBucketResponse.Buckets[0].BucketName; ListObjectsResponse listObjectsResponse = await localclient.ListObjectsAsync(bucketName); foreach (S3Object obj in listObjectsResponse.S3Objects) { Console.Out.WriteLine("key = '" + obj.Key + "'\t | size = " + obj.Size + "\t | tags = '" + obj.ETag + "'\t | modified = " + obj.LastModified); } } }
/// <summary> /// Verifica se existe a pasta da empresa no Bucket /// </summary> /// <param name="Pasta">Nome da pasta</param> /// <param name="BucketNome">Nome do bucket que a pasta deve estar</param> /// <returns>Retorna true caso a pasta existe e retorna false caso ela não exista</returns> public bool ExistePasta(string Pasta, string SubPasta, string BucketNome) { bool resp = false; //Objeto cliente que cria acesso com as chaves e define a região como São Paulo para maior otimização de upload IAmazonS3 cliente = new AmazonS3Client(ConfigurationManager.AppSettings["AWSAccessKeyS3"], ConfigurationManager.AppSettings["AWSSecretKeyS3"], RegionEndpoint.SAEast1); ListObjectsRequest findFolderRequest = new ListObjectsRequest(); findFolderRequest.BucketName = BucketNome; findFolderRequest.Prefix = Pasta + "/" + SubPasta; ListObjectsResponse findFolderResponse = cliente.ListObjects(findFolderRequest); //Caso exista o caminhono bucket retorna true, se não false resp = findFolderResponse.S3Objects.Any(); return(resp); }
public IDirectoryContents GetDirectoryContents(string subpath) { using (var client = GetClient()) { ListObjectsRequest request = new ListObjectsRequest(); request.BucketName = _setting.BucketName; ListObjectsResponse response = client.ListObjectsAsync(request).Result; List <IFileInfo> files = new List <IFileInfo>(); foreach (S3Object entry in response.S3Objects) { files.Add(new S3ObjectFileInfo(entry)); } return(new S3BucketDirectoryContents(files)); } }
// callback that handles retrieving a list of models on AWS. void ListModelsHandler(AmazonServiceResult <ListObjectsRequest, ListObjectsResponse> cb) { if (cb.Exception != null) { Debug.Log(cb.Exception); } string latestKey = ""; ListObjectsResponse resp = cb.Response; foreach (S3Object obj in resp.S3Objects) { latestKey = obj.Key; } // retrieve latest model and store it. Debug.Log("Importing model named " + latestKey); _client.GetObject(AWSClient.BUCKET_NAME, latestKey, GetModelObjectHandler); }
public async Task <List <S3Object> > listObjects(String bucket) { try { String key = "", secret = ""; getKeys(key, secret, "c:\\temp\\awskey.txt"); IAmazonS3 client = new AmazonS3Client(key, secret, Amazon.RegionEndpoint.USEast1); ListObjectsRequest request = new ListObjectsRequest(); request.BucketName = bucket; ListObjectsResponse response = await client.ListObjectsAsync(request); return(response.S3Objects); } catch (AmazonS3Exception amazonS3Exception) { throw amazonS3Exception; } }
/// <summary> /// 取得路径下所有文件的信息,不包括文件夹 /// </summary> /// <param name="bucketName"></param> /// <param name="dirName"></param> /// <returns></returns> public String ListDirOnlyFile(String bucketName, String dirName) { try { BosClient client = GenerateBosClient(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.BucketName = bucketName; listObjectsRequest.Delimiter = delimter; listObjectsRequest.Prefix = dirName; ListObjectsResponse response = client.ListObjects(listObjectsRequest); JArray jsonArray = new JArray(); //处理文件 foreach (BosObjectSummary obj in response.Contents) { String objName = obj.Key; if (objName.Equals(dirName)) { continue; //objName = "../"; } else if (dirName.Length != 0) { objName = objName.Replace(dirName, ""); } String objSize = Util.formatSize(obj.Size); String objDate = obj.LastModified.ToString("yyyy-MM-dd H:mm"); String objKey = obj.Key; JObject jsonObject = new JObject(); jsonObject.Add("objName", objName); jsonObject.Add("objSize", objSize); jsonObject.Add("objDate", objDate); jsonObject.Add("objKey", objKey); jsonObject.Add("objType", "F"); jsonArray.Add(jsonObject); } return(jsonArray.ToString()); } catch (Exception) { throw; } }
public async Task ListObjectsWithEncoding(S3Provider provider, string _, ISimpleClient client) { await CreateTempBucketAsync(provider, client, async tempBucket => { string tempObjName = "!#/()"; PutObjectResponse putResp = await client.PutObjectAsync(tempBucket, tempObjName, null).ConfigureAwait(false); Assert.Equal(200, putResp.StatusCode); ListObjectsResponse listResp = await client.ListObjectsAsync(tempBucket, r => r.EncodingType = EncodingType.Url).ConfigureAwait(false); Assert.Equal(200, listResp.StatusCode); Assert.Equal(EncodingType.Url, listResp.EncodingType); S3Object?obj = Assert.Single(listResp.Objects); Assert.Equal("%21%23/%28%29", obj.ObjectKey); }).ConfigureAwait(false); }
public static async Task SearchFromBucketWithBookIdAsync(AmazonS3Client client, string bookId) { ListObjectsRequest req = new ListObjectsRequest { BucketName = BUCKET_NAME }; ListObjectsResponse res = await client.ListObjectsAsync(req); foreach (S3Object obj in res.S3Objects) { var result = await SearchS3ObjectAsync(client, obj.Key).ConfigureAwait(true); if (result.bookid.ToString() == bookId.ToString()) { Console.WriteLine("Object Found with " + bookId); Console.WriteLine(JsonConvert.SerializeObject(result)); } } }
public List <S3Object> ListObjects(string prefix, Func <S3Object, bool> filterfFunc = null) { try { if (filterfFunc == null) { filterfFunc = (obj) => { return(true); }; } prefix = CleanPath(prefix); List <S3Object> result = new List <S3Object>(); ListObjectsRequest request = new ListObjectsRequest { BucketName = _amazonS3StorageConfiguration.AWSFileBucket, Prefix = prefix, MaxKeys = 1000 }; do { ListObjectsResponse response = _client.ListObjects(request); result.AddRange(response.S3Objects.Where(x => filterfFunc(x))); if (response.IsTruncated && response.S3Objects.Any()) { request.Marker = response.NextMarker; } else { break; } } while (request != null); return(result); } catch (AmazonS3Exception ex) { if (ex.StatusCode == HttpStatusCode.NotFound) { return(new List <S3Object>()); } throw; } }
public void CopyFolder(string sourceFolderPath, string targetFolderPath) { try { targetFolderPath = CorrectFolderPath(targetFolderPath); sourceFolderPath = CorrectFolderPath(sourceFolderPath); if (targetFolderPath == sourceFolderPath) { throw new ApplicationException("CopyFolder: target and source paths cannot be equal."); } // get entire contents of source folder ListObjectsRequest listRequest = new ListObjectsRequest() { BucketName = _bucketName, Prefix = sourceFolderPath }; ListObjectsResponse listResponse = _awsS3Client.ListObjects(listRequest); // create empty target folder CreateFolder(targetFolderPath); // copy contents of source to dest foreach (var sourceObject in listResponse.S3Objects) { CopyObjectRequest request = new CopyObjectRequest { SourceKey = sourceObject.Key, DestinationKey = sourceObject.Key.Replace(sourceFolderPath, targetFolderPath), SourceBucket = _bucketName, DestinationBucket = _bucketName, }; CopyObjectResponse response = _awsS3Client.CopyObject(request); } } catch (Exception ex) { throw new ApplicationException("S3 CopyFolder - unexpected exception.", ex); } }
public async Task GetFileList(string bucketName, string prefix = null) { try { ListObjectsRequest request = new ListObjectsRequest() { BucketName = bucketName, // list only things starting with "foo" Prefix = prefix // only list 3 things //MaxKeys = 3; }; //make the call string token = null; var result = new List <S3Object>(); do { ListObjectsResponse response = await client.ListObjectsAsync(request).ConfigureAwait(false); result.AddRange(response.S3Objects); token = response.NextMarker; } while (token != null); //loop thru results foreach (S3Object obj in result) { Console.WriteLine("key = {0} size = {1}", obj.Key, obj.Size); } } catch (AmazonS3Exception amazonS3Exception) { if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity"))) { Console.WriteLine("Please check the provided AWS Credentials."); Console.WriteLine("If you haven't signed up for Amazon S3, please visit http://aws.amazon.com/s3"); } else { Console.WriteLine("An error occurred with the message '{0}' when listing objects", amazonS3Exception.Message); } } }
public async Task <bool> Delete(OVEAssetModel asset) { _logger.LogInformation("about to delete file " + asset); try { using (var s3Client = GetS3Client(_configuration)) { // delete folder containing the file and everything ListObjectsResponse files = null; while (files == null || files.S3Objects.Any()) { if (files != null && files.S3Objects.Any()) { foreach (var o in files.S3Objects) { await s3Client.DeleteObjectAsync(asset.Project, o.Key); } } // find more files files = await s3Client.ListObjectsAsync(new ListObjectsRequest { BucketName = asset.Project, Prefix = asset.GetStorageGuid() }); } // if the bucket is empty then delete it var res = await s3Client.ListObjectsAsync(asset.Project); if (!res.S3Objects.Any()) { await s3Client.DeleteBucketAsync(asset.Project); } } _logger.LogInformation("deleted file on s3 correctly"); return(true); } catch (Exception e) { _logger.LogError(e, "Failed to delete an s3 file for " + asset); return(false); } }
/// <summary> /// バケット直下にディレクトリが存在しなければ、ディレクトリを作成する /// </summary> /// <param name="client">使用するS3Client</param> /// <param name="bucketName">バケット名</param> private async Task CreateDirUnderBucketAsync(AmazonS3Client client, string bucketName) { ListObjectsRequest request = new ListObjectsRequest { BucketName = bucketName, MaxKeys = 1000, Prefix = "", Delimiter = "/" // https://dev.classmethod.jp/cloud/aws/amazon-s3-folders/ }; try { LogDebug($"start querying objects under bucket"); // ディレクトリ・ファイルの一覧を取得 ListObjectsResponse response = await client.ListObjectsAsync(request); if (response.IsTruncated) { LogWarning("too many output files(should be less than 1000). exceeded files are ignored."); } LogDebug($"storeage response : {response.ToString()}"); // ディレクトリの存在チェックと作成 await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.Data.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingContainerOutputFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingContainerAttachedFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.TrainingHistoryAttachedFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.PreprocContainerAttachedFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceHistoryAttachedFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceContainerAttachedFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.InferenceContainerOutputFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.NotebookContainerAttachedFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.NotebookContainerOutputFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.ExperimentContainerAttachedFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.ExperimentContainerOutputFiles.ToString()); await CheckAndCreateDir(bucketName, client, response.CommonPrefixes, ResourceType.ExperimentPreprocContainerAttachedFiles.ToString()); } catch (Exception e) { LogDebug($"CreateDirUnderBucketAsync error : {e.ToString()}"); } }
private static List <S3Object> GetS3Objects(AmazonS3 client, string bucket, string prefix) { var request = new ListObjectsRequest().WithBucketName(bucket).WithPrefix(prefix); request.WithMaxKeys(1000); var objects = new List <S3Object>(); ListObjectsResponse response = null; do { response = client.ListObjects(request); response.S3Objects.ForEach(entry => objects.Add(entry)); if (objects.Count == 0) { return(objects); } request.Marker = objects[objects.Count - 1].Key; } while (response.IsTruncated); return(objects); }
public async Task ListObjectsWithDelimiter() { await CreateTempBucketAsync(async bucket => { string tempObjName = "object-" + Guid.NewGuid(); string tempObjName2 = "something-" + Guid.NewGuid(); await ObjectClient.PutObjectStringAsync(bucket, tempObjName, "hello").ConfigureAwait(false); await ObjectClient.PutObjectStringAsync(bucket, tempObjName2, "world!").ConfigureAwait(false); ListObjectsResponse resp = await ObjectClient.ListObjectsAsync(bucket, req => req.Delimiter = "-").ConfigureAwait(false); Assert.True(resp.IsSuccess); Assert.Equal("-", resp.Delimiter); Assert.Equal(2, resp.KeyCount); Assert.Equal(2, resp.CommonPrefixes.Count); Assert.Equal("object-", resp.CommonPrefixes[0]); Assert.Equal("something-", resp.CommonPrefixes[1]); }).ConfigureAwait(false); }
public static List <S3Object> GetAllImages() { var objectList = new List <S3Object>(); var client = new AmazonClient(); using (client.S3Client) { ListObjectsRequest request = new ListObjectsRequest { BucketName = client.BucketName, }; ListObjectsResponse response = client.S3Client.ListObjects(request); foreach (S3Object entry in response.S3Objects) { objectList.Add(entry); } } return(objectList); }
public override FileItem[] Search(FileItem from) { ListObjectsResponse objectListing = s3.ListObjects(from.BucketName, from.Path); var mFileItems = new List <FileItem>(); foreach (S3Object mS3Object in objectListing.S3Objects) { if (mS3Object.Key.Contains(from.SecondaryPath)) { FileItem mFileItem = new FileItem(); mFileItem.BucketName = mS3Object.BucketName; mFileItem.Path = mS3Object.Key; mFileItem.LastModified = mS3Object.LastModified.Ticks; mFileItem.Size = mS3Object.Size; mFileItems.Add(mFileItem); } } return(mFileItems.ToArray()); }
public override FileItem[] ListFiles(FileItem from) { ListObjectsResponse objectListing = s3.ListObjects(from.BucketName, from.Path); FileItem[] mFileItems = new FileItem[objectListing.S3Objects.Count]; int i = 0; foreach (S3Object mS3Object in objectListing.S3Objects) { FileItem mFileItem = new FileItem(); mFileItem.BucketName = mS3Object.BucketName; mFileItem.Path = mS3Object.Key; mFileItem.IsDirectory = mS3Object.Key.EndsWith("/"); mFileItem.LastModified = mS3Object.LastModified.Ticks; mFileItem.Size = mS3Object.Size; mFileItems[i] = mFileItem; i++; } return(mFileItems); }
public ActionResult Index() { // get me all objects inside a given folder Dictionary <string, double> images = null; var request = new ListObjectsRequest(); request.BucketName = AWSBucket; request.WithPrefix(AWSFolder); using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(AWSAccessKey, AWSSecretKey)) { using (ListObjectsResponse response = client.ListObjects(request)) { images = response.S3Objects.Where(x => x.Key != AWSFolder).ToDictionary(obj => obj.Key, obj => AppHelper.ConvertBytesToMegabytes(obj.Size)); } } return(View(images)); }