public async Task DeleteRecursiveAsync(string fullPath, CancellationToken cancellationToken)
        {
            ListObjectsV2Request request = new ListObjectsV2Request
            {
                BucketName = this._bucketName, Prefix = fullPath + "/"
            };

            while (true)
            {
                ListObjectsV2Response response =
                    await this._client.ListObjectsV2Async(request, cancellationToken).ConfigureAwait(false);

                await Task.WhenAll(
                    response.S3Objects.Select(
                        s3 =>
                        this._client.DeleteObjectAsync(this._bucketName, s3.Key, cancellationToken)));

                if (response.NextContinuationToken != null)
                {
                    request.ContinuationToken = response.NextContinuationToken;
                }
                else
                {
                    break;
                }
            }
        }
Beispiel #2
0
        /// <summary>
        /// Lists all buckets, optionaly filtering by prefix. Prefix filtering happens on client side.
        /// </summary>
        public async Task <IEnumerable <BlobId> > ListAsync(ListOptions options, CancellationToken cancellationToken)
        {
            if (options == null)
            {
                options = new ListOptions();
            }

            GenericValidation.CheckBlobPrefix(options.Prefix);

            var request = new ListObjectsV2Request()
            {
                BucketName = _bucketName,
                Prefix     = options.Prefix ?? null
            };

            if (options.MaxResults.HasValue)
            {
                request.MaxKeys = options.MaxResults.Value;
            }


            //todo: paging
            ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken);

            return(response.S3Objects
                   .Select(s3Obj => new BlobId(StoragePath.RootFolderPath, s3Obj.Key, BlobItemKind.File)));
        }
Beispiel #3
0
        static async Task CreateFoldersAsync(string bucketName, string path)
        {
            var findFolderRequest = new ListObjectsV2Request();

            findFolderRequest.BucketName = bucketName;
            findFolderRequest.Prefix     = path;

            ListObjectsV2Response findFolderResponse =
                await s3Client.ListObjectsV2Async(findFolderRequest);


            if (findFolderResponse.S3Objects.Any())
            {
                return;
            }

            PutObjectRequest request = new PutObjectRequest()
            {
                BucketName  = bucketName,
                Key         = path,
                ContentBody = string.Empty
            };

            // add try catch in case you have exceptions shield/handling here
            PutObjectResponse response = await s3Client.PutObjectAsync(request);
        }
Beispiel #4
0
        internal void RenameFolder(String sourceKey, String destKey)
        {
            String fullSource = GetFullPrefix(sourceKey);
            String fullDest   = GetFullPrefix(destKey);

            ListObjectsV2Request request = new ListObjectsV2Request()
            {
                BucketName = _bucketName,
                FetchOwner = false,
                Prefix     = VirtualPathUtility.AppendTrailingSlash(fullSource),
            };

            ListObjectsV2Response response = null;

            do
            {
                if (response != null && response.IsTruncated)
                {
                    request.ContinuationToken = response.NextContinuationToken;
                }

                response = _S3Client.ListObjectsV2(request);

                foreach (S3Object s3Object in response.S3Objects)
                {
                    String newKey = fullDest + s3Object.Key.Substring(fullSource.Length);
                    RenameObject(s3Object.Key, newKey);
                }
            } while (response != null && response.IsTruncated);
        }
Beispiel #5
0
        public async Task <IEnumerable <TaggedImage> > GetAllImages()
        {
            using (var scope = _logger.BeginScope(nameof(GetAllImages)))
            {
                var request = new ListObjectsV2Request
                {
                    BucketName = _s3BucketName
                };
                ListObjectsV2Response response = null;
                var taggedFiles = new List <TaggedImage>();
                do
                {
                    response = await _s3Client.ListObjectsV2Async(request);

                    _logger.LogDebug($"Response: {JsonConvert.SerializeObject(response)}");

                    var batch = response.S3Objects
                                .Where(o => IsSupportedImageFormat(o.Key))
                                .Select(o => GetTaggedImage(o).Result);

                    taggedFiles.AddRange(batch);
                } while (response.HttpStatusCode == HttpStatusCode.OK && response.IsTruncated);

                return(taggedFiles);
            }
        }
        public ListObjectsV2Response ListObjectsByPrefixOrDelimiter(string bucketName, string key, string delimiter = null)
        {
            try
            {
                var request = new ListObjectsV2Request()
                {
                    BucketName = bucketName, Prefix = key
                };
                if (delimiter != null)
                {
                    request.Delimiter = delimiter;
                }

                var listObjectV2Response = new ListObjectsV2Response();
                do
                {
                    var response = _amazonS3.ListObjectsV2Async(request).Result;
                    if (response == null || response?.KeyCount == 0)
                    {
                        return(listObjectV2Response);
                    }

                    listObjectV2Response.IsTruncated = response.IsTruncated;
                    listObjectV2Response.S3Objects.AddRange(response.S3Objects);
                } while (listObjectV2Response.IsTruncated);

                return(listObjectV2Response);
            }
            catch (Exception ex)
            {
                Console.WriteLine(JsonConvert.SerializeObject(ex));
                throw ex;
            }
        }
Beispiel #7
0
        /// <summary>
        /// GetListing generates a list of <see cref="T:Amazon.S3.Model.S3Object" /> within a certain prefix, without recursing by default (folder specific listing)
        /// </summary>
        /// <param name="prefix">Prefix to list</param>
        /// <param name="recursive">Request recursive listing under folder</param>
        /// <returns>List of <see cref="T:Amazon.S3.Model.S3Object" /></returns>
        /// <remarks>In Amazon S3, folders are non-existing items. This function generates pseudo-items for folders.</remarks>
        internal IEnumerable <S3ItemData> GetListing(String prefix, bool recursive = false)
        {
            ListObjectsV2Request request = new ListObjectsV2Request()
            {
                BucketName = _bucketName,
                FetchOwner = false,
                Prefix     = VirtualPathUtility.AppendTrailingSlash(GetFullPrefix(prefix)),
                Delimiter  = recursive ? null : "/"
            };

            ListObjectsV2Response response = null;

            do
            {
                if (response != null && response.IsTruncated)
                {
                    request.ContinuationToken = response.NextContinuationToken;
                }

                response = _S3Client.ListObjectsV2(request);

                foreach (String subPrefix in response.CommonPrefixes)
                {
                    yield return(GetFolder(StripPrefix(subPrefix)));
                }

                foreach (S3Object s3Object in response.S3Objects)
                {
                    yield return(new S3ItemData(StripPrefix(s3Object.Key), _bucketUrl, s3Object));
                }
            } while (response != null && response.IsTruncated);
        }
Beispiel #8
0
        public void GetAllIdsReturnsAllIdsFromListObjectsV2Test()
        {
            var response = new ListObjectsV2Response
            {
                S3Objects = new List <S3Object>()
                {
                    new S3Object {
                        Key = "keyprefix/1"
                    },
                    new S3Object {
                        Key = "keyprefix/2"
                    },
                    new S3Object {
                        Key = "keyprefix/3"
                    }
                }
            };

            _mockS3Client
            .Setup(
                x => x.ListObjectsV2Async(
                    It.Is <ListObjectsV2Request>(
                        y => y.BucketName == "mybucket" && y.Prefix == "keyprefix/" && y.Delimiter == "/"),
                    It.IsAny <CancellationToken>()))
            .ReturnsAsync(response);

            var result = _store.GetAllIds().Result;

            Assert.Equal(3, result.Count);
            Assert.Contains(1, result);
            Assert.Contains(2, result);
            Assert.Contains(3, result);
        }
Beispiel #9
0
        public async Task <IReadOnlyCollection <XElement> > GetAllElementsAsync(CancellationToken cancellation)
        {
            var items = new List <S3Object>();
            ListObjectsV2Response response = null;

            do
            {
                response = await _client.ListObjectsV2Async(new ListObjectsV2Request
                {
                    BucketName        = Config.Bucket,
                    Prefix            = Config.KeyPrefix,
                    ContinuationToken = response?.NextContinuationToken
                }, cancellation);

                items.AddRange(response.S3Objects);
            } while (response.IsTruncated);

            using (var throttler = new SemaphoreSlim(Config.MaxS3QueryConcurrency))
            {
                var queries = new List <Task <XElement> >();
                foreach (var item in items)
                {
                    queries.Add(GetElementFromKeyAsync(item, throttler, cancellation));
                }

                await Task.WhenAll(queries).ConfigureAwait(false);

                return(new ReadOnlyCollection <XElement>(queries.Select(x => x.Result).Where(x => x != null).ToList()));
            }
        }
Beispiel #10
0
        private async Task <ListObjectsV2Response> ListContainerObjects(string folderKey, bool useDelimiter = true)
        {
            var request = new ListObjectsV2Request
            {
                BucketName = _amazonS3Config.Bucket,
                Prefix     = $"{folderKey}/",
                Delimiter  = useDelimiter ? "/" : string.Empty
            };

            var response = new ListObjectsV2Response();

            var paginatedResponse = new ListObjectsV2Response
            {
                IsTruncated = true
            };

            while (paginatedResponse.IsTruncated)
            {
                request.ContinuationToken = paginatedResponse.NextContinuationToken;
                paginatedResponse         = await _amazonS3.ListObjectsV2Async(request);

                response.S3Objects.AddRange(paginatedResponse.S3Objects);
                response.CommonPrefixes.AddRange(paginatedResponse.CommonPrefixes);
            }

            return(response);
        }
Beispiel #11
0
        /// <summary>
        /// Uses the client object to get a list of the objects in the Amazon
        /// S3 bucket in the bucketName parameter.
        /// </summary>
        /// <param name="client">The initialized Amazon S3 client object used to call
        /// the ListObjectsAsync method.</param>
        /// <param name="bucketName">The bucket name for which you want to
        /// retrieve a list of objects.</param>
        public static async Task ListingObjectsAsync(IAmazonS3 client, string bucketName)
        {
            try
            {
                ListObjectsV2Request request = new()
                {
                    BucketName = bucketName,
                    MaxKeys    = 5,
                };

                var response = new ListObjectsV2Response();

                do
                {
                    response = await client.ListObjectsV2Async(request);

                    response.S3Objects
                    .ForEach(obj => Console.WriteLine($"{obj.Key,-35}{obj.LastModified.ToShortDateString(),10}{obj.Size,10}"));

                    // If the response is truncated, set the request ContinuationToken
                    // from the NextContinuationToken property of the response.
                    request.ContinuationToken = response.NextContinuationToken;
                } while (response.IsTruncated);
            }
            catch (AmazonS3Exception ex)
            {
                Console.WriteLine($"Error encountered on server. Message:'{ex.Message}' getting list of objects.");
            }
        }
        public async Task <List <string> > GetCensusFiles()
        {
            var request = new ListObjectsV2Request
            {
                BucketName = environmentModel.Bucket,
                Prefix     = environmentModel.KeyPrefix
            };

            ListObjectsV2Response listResponse = null;

            listResponse = await awsClient.ListObjectsV2Async(request);

            var keys = listResponse.S3Objects
                       .Select(a => a.Key)
                       .ToList();

            while (listResponse.IsTruncated)
            {
                request = new ListObjectsV2Request
                {
                    BucketName        = environmentModel.Bucket,
                    ContinuationToken = listResponse.NextContinuationToken
                };

                listResponse = await awsClient.ListObjectsV2Async(request);

                var newKeys = listResponse.S3Objects
                              .Select(a => a.Key)
                              .ToList();

                keys.AddRange(newKeys);
            }

            return(keys);
        }
        private Batch ProcessResponse(int batchId, ListObjectsV2Response response)
        {
            // move to the next batch
            Interlocked.Exchange(ref _continuationToken, response.NextContinuationToken);
            Interlocked.Exchange(ref _startAtKey, null);
            // build the batch for the target objects
            Batch batch = new Batch(_prefix, batchId, _stopAtKey, response);

            // we should only ever add batches whose key ranges are after the ones we already had
            if (!string.IsNullOrEmpty(_unprefixedGreatestKey) && String.CompareOrdinal(batch.LeastKey, _unprefixedGreatestKey) < 0)
            {
                throw new InvalidOperationException("The specified response is for objects that are lesser than the ones already processing.  Responses must be processed in order (" + batch.LeastKey + "/" + _unprefixedGreatestKey + ").");
            }
            if (!string.IsNullOrEmpty(_unprefixedGreatestKey) && String.CompareOrdinal(batch.GreatestKey, _unprefixedGreatestKey) < 0)
            {
                throw new InvalidOperationException("The specified response is for objects that are lesser than the ones already processing.  Responses must be processed in order (" + batch.GreatestKey + "/" + _unprefixedGreatestKey + ").");
            }
            // no least key specified yet? use the one from the batch
            if (string.IsNullOrEmpty(_unprefixedLeastKey))
            {
                Interlocked.Exchange(ref _unprefixedLeastKey, batch.LeastKey);
            }
            // the greatest key should be the one from this new batch
            Interlocked.Exchange(ref _unprefixedGreatestKey, batch.GreatestKey);
            // add this batch to the queue
            _queue.Enqueue(batch);
            Interlocked.Exchange(ref _lastQueuedBatchId, batchId);
            // is this the last batch?
            Interlocked.Exchange(ref _lastBatchHasBeenRead, batch.Response.IsTruncated ? 0 : 1);
            return(batch);
        }
        public void ExpectEmptyQueryToSucceed()
        {
            var listResponse = new ListObjectsV2Response
            {
                Name   = Bucket,
                Prefix = Prefix
            };

            config.Setup(x => x.Bucket).Returns(Bucket);
            config.Setup(x => x.KeyPrefix).Returns(Prefix);
            config.Setup(x => x.MaxS3QueryConcurrency).Returns(10);

            s3Client.Setup(x => x.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), CancellationToken.None))
            .ReturnsAsync(listResponse)
            .Callback <ListObjectsV2Request, CancellationToken>((lr, ct) =>
            {
                Assert.Equal(Bucket, lr.BucketName);
                Assert.Equal(Prefix, lr.Prefix);
                Assert.Null(lr.ContinuationToken);
            });

            IReadOnlyCollection <XElement> list = xmlRepository.GetAllElements();

            Assert.Equal(0, list.Count);
        }
        private Batch GetNextBatch()
        {
            // already queued the last batch? return an indicator that we're at the end!
            if (_lastBatchHasBeenRead != 0)
            {
                return(null);
            }
            ListObjectsV2Request request = BuildRequest();

            Program.State.AddChargeForQueries(1);
            int    batchId   = _batchIdCounter.Next;
            string operation = "QUERY: Get batch " + batchId + " from " + _bucket + "/" + _prefix;

            Interlocked.Exchange(ref _lastQuery, operation);
            Stopwatch             timer    = Stopwatch.StartNew();
            ListObjectsV2Response response = null;

            using (Program.TrackOperation(operation))
            {
                response = _s3.ListObjectsV2(request);
            }
            if (!response.IsTruncated)
            {
                Program.LogVerbose("Last batch from " + _bucket + "/" + _prefix + "!");
            }
            using (Program.TrackOperation("QUERY: processing batch " + batchId + " from " + _bucket + "/" + _prefix + " (" + response.S3Objects.Count + ")"))
            {
                return(ProcessResponse(batchId, response));
            }
        }
        private S3Object getFile()
        {
            AmazonS3Client       client  = getConfig();
            ListObjectsV2Request request = new ListObjectsV2Request()
            {
                BucketName = _bucketName
            };
            ListObjectsV2Response list = client.ListObjectsV2(request);
            long     data = 0;
            S3Object file = new S3Object();

            foreach (var item in list.S3Objects)
            {
                if (data < item.Size)
                {
                    file = item;
                    data = item.Size;
                }
            }
            // Convert Bytes to MB
            double t = (file.Size / 1024f) / 1024f;

            // Check how many files we need to achieve x quantity of GB (This case 4.9GB)
            t = 4900 / t;
            // Round number and converted to Int
            cantidad = Convert.ToInt32(Math.Round(t));
            // Return S3 Object
            return(file);
        }
Beispiel #17
0
        public void ListBucketContents()
        {
            var andrew = "";
            var secret = "";

            //this.bucket = "stuff.tiz.digital";
            AmazonS3Client client = new AmazonS3Client(andrew, secret, RegionEndpoint.APSoutheast2);

            // Issue call
            ListBucketsResponse response = client.ListBuckets();

            // View response data
            Debug.WriteLine($"Buckets owner - {response.Owner.DisplayName}");
            foreach (S3Bucket bucket in response.Buckets)
            {
                Debug.WriteLine($"{bucket.BucketName}, Created {bucket.CreationDate}");
            }

            ListObjectsV2Request objRequest = new ListObjectsV2Request
            {
                BucketName = "rmccomb"
            };

            ListObjectsV2Response objResponse = client.ListObjectsV2(objRequest);

            foreach (S3Object obj in objResponse.S3Objects)
            {
                Debug.WriteLine($"{obj.BucketName} {obj.Key}");
            }
        }
Beispiel #18
0
        public async Task <List <string> > IndexSchema()
        {
            if (!_distributedCacheConfiguration.UseDistributedCache)
            {
                _logger.LogWarning($"S3FileSchemaProvider::IndexSchema, A request to index form schema was made but UseDistributedCache is disabled");
                return(new List <string>());
            }

            var result = new ListObjectsV2Response();

            try
            {
                result = await _s3Gateway.ListObjectsV2(_configuration["S3BucketKey"], $"{_environment.EnvironmentName.ToS3EnvPrefix()}/{_configuration["ApplicationVersion"]}");
            }
            catch (Exception e)
            {
                _logger.LogWarning($"S3FileSchemaProvider::IndexSchema, Failed to retrieve list of forms from s3 bucket {_environment.EnvironmentName.ToS3EnvPrefix()}/{_configuration["ApplicationVersion"]}, Exception: {e.Message}");
                return(new List <string>());
            }

            var indexKeys = result.S3Objects.Select(_ => _.Key).ToList();

            _ = _distributedCacheWrapper.SetStringAsync(CacheConstants.INDEX_SCHEMA, JsonConvert.SerializeObject(indexKeys), _distributedCacheExpirationConfiguration.Index);

            return(indexKeys);
        }
        static void ListBucket()
        {
            string continuationToken = Common.InputString("Continuation token:", null, true);

            ListObjectsV2Request request = new ListObjectsV2Request();

            request.BucketName        = _Bucket;
            request.ContinuationToken = continuationToken;

            ListObjectsV2Response response = _S3Client.ListObjectsV2Async(request).Result;

            if (response != null)
            {
                foreach (S3Object curr in response.S3Objects)
                {
                    Console.WriteLine(curr.Key + ": " + curr.Size + " bytes");
                }

                if (!String.IsNullOrEmpty(response.NextContinuationToken))
                {
                    Console.WriteLine("Continuation token: " + response.NextContinuationToken);
                }

                Console.WriteLine("Success");
            }
            else
            {
                Console.WriteLine("Failed");
            }
        }
Beispiel #20
0
        public async Task <IList <AwsFileInfo> > GetBucketFiles(string bucketName)
        {
            client = new AmazonS3Client(bucketRegion);
            try
            {
                ListObjectsV2Request request = new ListObjectsV2Request
                {
                    BucketName = bucketName,
                    MaxKeys    = 50
                };
                ListObjectsV2Response response = await client.ListObjectsV2Async(request);

                return(response.S3Objects.Select(x => new AwsFileInfo
                {
                    FileName = x.Key,
                    Size = x.Size
                }).ToList());

                // request.ContinuationToken = response.NextContinuationToken;
                // while (response.IsTruncated);
            }
            catch (AmazonS3Exception amazonS3Exception)
            {
                throw new Exception("S3 error occurred. Exception: " + amazonS3Exception.ToString());
            }
            catch (Exception e)
            {
                throw new Exception("Exception: " + e.ToString());
            }
        }
        /// <summary>
        /// Lists all buckets, optionaly filtering by prefix. Prefix filtering happens on client side.
        /// </summary>
        public async Task <IReadOnlyCollection <BlobId> > ListAsync(ListOptions options = null, CancellationToken cancellationToken = default)
        {
            if (options == null)
            {
                options = new ListOptions();
            }

            GenericValidation.CheckBlobPrefix(options.FilePrefix);

            var request = new ListObjectsV2Request()
            {
                BucketName = _bucketName,
                Prefix     = options.FilePrefix ?? null
            };

            if (options.MaxResults.HasValue)
            {
                request.MaxKeys = options.MaxResults.Value;
            }


            //todo: paging
            AmazonS3Client client = await GetClientAsync();

            ListObjectsV2Response response = await client.ListObjectsV2Async(request, cancellationToken);

            return(response.S3Objects
                   .Select(s3Obj => new BlobId(StoragePath.RootFolderPath, s3Obj.Key, BlobItemKind.File))
                   .Where(options.IsMatch)
                   .Where(bid => (options.FolderPath == null || bid.FolderPath == options.FolderPath))
                   .Where(bid => options.BrowseFilter == null || options.BrowseFilter(bid))
                   .ToList());
        }
Beispiel #22
0
        public List <DocumentEntity> ListDocuments(string username)
        {
            List <DocumentEntity> documents = new List <DocumentEntity>();

            SetS3Client();

            ListObjectsV2Request request = new ListObjectsV2Request()
            {
                BucketName = this.BucketName,
                Prefix     = username
            };

            try
            {
                ListObjectsV2Response response = s3Client.ListObjectsV2(request);

                if (response.S3Objects.Count > 0)
                {
                    foreach (S3Object s3object in response.S3Objects)
                    {
                        documents.Add(ConvertS3ObjectToDocumentEntity(s3object));
                    }
                }
            }
            catch (Exception ex)
            {
                throw ex;
            }

            return(documents);
        }
Beispiel #23
0
        public async Task <IActionResult> ListFiles(string bucketName)
        {
            string continuationToken        = null;
            List <ListFilesResponse> result = new List <ListFilesResponse>();

            do
            {
                ListObjectsV2Response response = await _s3Client.ListObjectsV2Async(new ListObjectsV2Request
                {
                    BucketName        = bucketName,
                    ContinuationToken = continuationToken
                });

                if (response.HttpStatusCode != System.Net.HttpStatusCode.OK)
                {
                    return(StatusCode((int)response.HttpStatusCode));
                }
                continuationToken = response.ContinuationToken;
                if (response.S3Objects != null && response.S3Objects.Count > 0)
                {
                    result.AddRange(response.S3Objects.Select(c => new ListFilesResponse
                    {
                        BucketName   = c.BucketName,
                        Key          = c.Key,
                        Size         = c.Size,
                        LastModified = c.LastModified,
                        Owner        = c.Owner?.DisplayName
                    }));
                }
            } while (continuationToken != null);


            return(Ok(result));
        }
Beispiel #24
0
        public async Task <AwsListResponse> GetListAsync(int take, string startKey = "")
        {
            AwsListResponse     awsResponses = new AwsListResponse();
            AwsItemListResponse awsResponse  = new AwsItemListResponse();

            try
            {
                ListObjectsV2Request request = new ListObjectsV2Request
                {
                    BucketName = BucketPath,
                    StartAfter = startKey,
                    MaxKeys    = take
                };

                ListObjectsV2Response response = await _client.ListObjectsV2Async(request);

                awsResponse.RequestId      = response.ResponseMetadata.RequestId;
                awsResponse.HttpStatusCode = response.HttpStatusCode;

                do
                {
                    response = await _client.ListObjectsV2Async(request);

                    awsResponse = new AwsItemListResponse()
                    {
                        RequestId      = response.ResponseMetadata.RequestId,
                        HttpStatusCode = response.HttpStatusCode
                    };

                    awsResponses.Responses.Add(awsResponse);

                    foreach (S3Object entry in response.S3Objects)
                    {
                        awsResponse.Items.Add(new AwsListItem()
                        {
                            Key          = entry.Key,
                            ETag         = entry.ETag,
                            LastModified = entry.LastModified
                        });
                    }

                    Console.WriteLine("Next Continuation Token: {0}", response.NextContinuationToken);

                    request.ContinuationToken = response.NextContinuationToken;
                } while (response.IsTruncated);
            }
            catch (AmazonS3Exception amazonS3Exception)
            {
                awsResponses.Exception = amazonS3Exception;
            }
            catch (Exception e)
            {
                awsResponses.Exception = e;
            }

            return(awsResponses);
        }
Beispiel #25
0
        private static async Task <IEnumerable <string> > TransformData(IAmazonS3 client)
        {
            string outputBucket = _config["OutputBucket"];

            await Setup(client);

            ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request();

            listObjectsRequest.BucketName = _config["InputBucket"];

            ListObjectsV2Response listResponse = client.ListObjectsV2Async(listObjectsRequest).Result;

            if (listResponse.KeyCount == 0)
            {
                Console.WriteLine("No items found in input bucket");
                return(null);
            }
            else
            {
                Console.WriteLine($"Transforming {listResponse.KeyCount} objects");
            }

            List <string> urls = new List <string>();

            foreach (S3Object inputItem in listResponse.S3Objects)
            {
                GetObjectResponse response =
                    await client.GetObjectAsync(
                        inputItem.BucketName,
                        inputItem.Key
                        );

                PutObjectRequest putRequest = new PutObjectRequest();
                putRequest.BucketName = outputBucket;

                string outputKey = $"{inputItem.Key.Split("-")[0]}-output";

                using (TextReader reader = new StreamReader(response.ResponseStream))
                {
                    putRequest.ContentBody = Convert.ToBase64String(Encoding.UTF8.GetBytes(await reader.ReadToEndAsync()));
                    putRequest.ContentType = "text/plain";
                    putRequest.Key         = outputKey;
                }

                await client.PutObjectAsync(putRequest);

                GetPreSignedUrlRequest presignedRequest = new GetPreSignedUrlRequest();
                presignedRequest.BucketName = outputBucket;
                presignedRequest.Key        = outputKey;
                presignedRequest.Expires    = DateTime.Now.AddMinutes(30);

                urls.Add(client.GetPreSignedURL(presignedRequest));
            }

            return(urls);
        }
Beispiel #26
0
        public static IReadOnlyCollection <Blob> ToBlobs(
            this ListObjectsV2Response response,
            ListOptions options)
        {
            List <Blob> blobList = new List <Blob>();

            blobList.AddRange(response.S3Objects.Where(b => !b.Key.EndsWith("/")).Select(b => b.ToBlob()).Where(options.IsMatch).Where(b => options.BrowseFilter == null || options.BrowseFilter(b)));
            blobList.AddRange(response.CommonPrefixes.Select(p => new Blob(p, BlobItemKind.Folder)));
            return(blobList);
        }
Beispiel #27
0
        public async Task <List <S3Object> > ListFilesInS3(IAmazonS3 s3Client, string bucketName, string keyPrefix)
        {
            ListObjectsV2Response response = await s3Client.ListObjectsV2Async(new ListObjectsV2Request()
            {
                BucketName = bucketName,
                Prefix     = keyPrefix
            });

            return(response.S3Objects);
        }
Beispiel #28
0
        private async Task <string> GetNewDirectoryPathIfExists(string folderKey, string folderKeyToCheck, int number = 0)
        {
            ListObjectsV2Response response = await ListContainerObjects(folderKeyToCheck);

            if (response.S3Objects.Count <= 0)
            {
                return(folderKeyToCheck);
            }
            return(await GetNewDirectoryPathIfExists(folderKey, $"{folderKey} - Copy {(number > 1 ? $"({number + 1})" : "")}", number + 1));
        }
Beispiel #29
0
        public static async Task Main(string[] args)
        {
            var client = new AmazonS3Client(RegionEndpoint.EUWest2);
            ListBucketsResponse buckets_response = await client.ListBucketsAsync();

            foreach (var bucket in buckets_response.Buckets)
            {
                ListObjectsV2Request req = new ListObjectsV2Request();
                req.BucketName = bucket.BucketName;
                // NOTE if this finds a folder it includes the folder as an object, and its content as
                // more objects.  I thought folders didn't exist as such, but it seems they do.
                ListObjectsV2Response res = await client.ListObjectsV2Async(req);

                int len = res.S3Objects.Count;
                Console.WriteLine(bucket.BucketName + " date:" + bucket.CreationDate + " has " + len + " objects.");
                if (len > 0)
                {
                    foreach (var obj in res.S3Objects)
                    {
                        Console.WriteLine("   " + obj.Key + " " + obj.Size + " " + obj.StorageClass + " " + obj.ETag);
                    }
                }
            }

            var pollyclient = new AmazonPollyClient();
            SynthesizeSpeechRequest request = new SynthesizeSpeechRequest();

            request.Text         = "Hello my lovely little world.";
            request.OutputFormat = OutputFormat.Mp3;
            request.VoiceId      = VoiceId.Emma;
            var pollyres = await pollyclient.SynthesizeSpeechAsync(request);

            var fs    = new FileStream("c:/tmp/hw.mp3", FileMode.Create);
            var instr = pollyres.AudioStream;

            byte[] bytes = new byte[256];
            int    count = 0;

            while (true) //count < instr.Length)
            {
                int c = instr.Read(bytes, 0, 256);
                if (c == 0)
                {
                    break;
                }
                fs.Write(bytes, 0, c);
                count += c;
            }
            fs.Close();
            var okater = new NetCoreAudio.Player();

            okater.Play("c:/tmp/hw.mp3").Wait();
            Console.WriteLine("Thanks for all the fish");
            Console.ReadLine();
        }
Beispiel #30
0
        public void fetchKeys()
        {
            request = new ListObjectsV2Request
            {
                BucketName = BucketName,
                MaxKeys    = 5
            };

            response   = client.ListObjectsV2(request);
            enumerator = response.S3Objects.GetEnumerator();
        }