/// <summary> /// Gets the objects in folder without recursion. i.e. will get the list of files /// and folders in the folder but not the contents of the subfolders. Subfolders /// will not have the ModifiedDate prop filled in as Amazon doesn't provide it in /// this context. /// </summary> /// <param name="assetStorageProvider">The asset storage Provider.</param> /// <param name="asset">The asset.</param> /// <returns></returns> public override List <Asset> ListObjectsInFolder(AssetStorageProvider assetStorageProvider, Asset asset) { string rootFolder = FixRootFolder(GetAttributeValue(assetStorageProvider, AttributeKeys.RootFolder)); string bucketName = GetAttributeValue(assetStorageProvider, AttributeKeys.Bucket); asset.Key = FixKey(asset, rootFolder); HasRequirementsFolder(asset); try { AmazonS3Client client = GetAmazonS3Client(assetStorageProvider); ListObjectsV2Request request = new ListObjectsV2Request(); request.BucketName = bucketName; request.Prefix = asset.Key == "/" ? rootFolder : asset.Key; request.Delimiter = "/"; var assets = new List <Asset>(); var subFolders = new HashSet <string>(); ListObjectsV2Response response; // S3 will only return 1,000 keys per response and sets IsTruncated = true, the do-while loop will run and fetch keys until IsTruncated = false. do { response = client.ListObjectsV2(request); foreach (S3Object s3Object in response.S3Objects) { if (s3Object.Key == null) { continue; } var responseAsset = CreateAssetFromS3Object(assetStorageProvider, s3Object, client.Config.RegionEndpoint.SystemName); assets.Add(responseAsset); } // After setting the delimiter S3 will filter out any prefixes below that in response.S3Objects. // So we need to inspect response.CommonPrefixes to get the prefixes inside the folder. foreach (string subFolder in response.CommonPrefixes) { if (subFolder.IsNotNullOrWhiteSpace()) { subFolders.Add(subFolder); } } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); // Add the subfolders to the asset collection foreach (string subFolder in subFolders) { var subFolderAsset = CreateAssetFromCommonPrefix(subFolder, client.Config.RegionEndpoint.SystemName, bucketName); assets.Add(subFolderAsset); } return(assets.OrderBy(a => a.Key, StringComparer.OrdinalIgnoreCase).ToList()); } catch (Exception ex) { ExceptionLogService.LogException(ex); throw; } }
private async Task ShowFailedKinesisProcessingLogs(Settings settings, string keyPrefix) { // populate information about the deployment tier await PopulateDeploymentTierSettingsAsync(settings); if (settings.LoggingBucketName == null) { LogError("Deployment tier does not a have logging bucket"); return; } // list all objects under the specified key var request = new ListObjectsV2Request { BucketName = settings.LoggingBucketName, Prefix = keyPrefix, MaxKeys = 100 }; do { var response = await settings.S3Client.ListObjectsV2Async(request); foreach (var s3Object in response.S3Objects) { Console.WriteLine($"{Settings.OutputColor}Key:{Settings.ResetColor}: {s3Object.Key}"); try { var records = await GetS3ObjectContentsAsync(s3Object.Key); foreach (var record in records) { Console.WriteLine($"{Settings.InfoColor}ArrivalTime:{Settings.ResetColor} {DateTimeOffset.FromUnixTimeMilliseconds(record.arrivalTimestamp)}"); Console.WriteLine($"{Settings.InfoColor}AttemptEndingTimestamp:{Settings.ResetColor} {DateTimeOffset.FromUnixTimeMilliseconds(record.attemptEndingTimestamp)}"); Console.WriteLine($"{Settings.InfoColor}AttemptsMade:{Settings.ResetColor} {record.attemptsMade}"); Console.WriteLine($"{Settings.InfoColor}ErrorCode:{Settings.ResetColor} {record.errorCode}"); Console.WriteLine($"{Settings.InfoColor}ErrorMessage:{Settings.ResetColor} {record.errorMessage}"); Console.WriteLine($"{Settings.InfoColor}Lambda ARN:{Settings.ResetColor} {record.lambdaArn}"); var entries = await DecodeBase64GzipDataAsync(record.rawData); Console.WriteLine($"{Settings.InfoColor}Entries:{Settings.ResetColor} {JObject.Parse(entries).ToString(Formatting.Indented)}"); } } catch (Exception e) { LogError($"unable to read records for {s3Object.Key}", e); } Console.WriteLine(); } request.ContinuationToken = response.NextContinuationToken; } while(request.ContinuationToken != null); // local functions async Task <IEnumerable <KinesisFailedLogRecord> > GetS3ObjectContentsAsync(string key) { try { var response = await settings.S3Client.GetObjectAsync(new GetObjectRequest { BucketName = settings.LoggingBucketName, Key = key }); using (var decompressionStream = new GZipStream(response.ResponseStream, CompressionMode.Decompress)) using (var destinationStream = new MemoryStream()) { await decompressionStream.CopyToAsync(destinationStream); return(Encoding.UTF8.GetString(destinationStream.ToArray()) .Split(SEPARATORS, StringSplitOptions.RemoveEmptyEntries) .Select(json => JsonConvert.DeserializeObject <KinesisFailedLogRecord>(json.Trim()))); } } catch (AmazonS3Exception) { return(null); } } async Task <string> DecodeBase64GzipDataAsync(string data) { using (var sourceStream = new MemoryStream(Convert.FromBase64String(data))) using (var decompressionStream = new GZipStream(sourceStream, CompressionMode.Decompress)) using (var destinationStram = new MemoryStream()) { await decompressionStream.CopyToAsync(destinationStram); return(Encoding.UTF8.GetString(destinationStram.ToArray())); } } }
public async Task <IEnumerable <StorageProviderItem> > GetChildrenByParentItem(StorageProviderItem parent) { using (var api = AmazonS3Helper.GetApi(_account)) { if (parent.Id == "/") { var response = await api.ListBucketsAsync(); var items = response.Buckets.Select(_ => new StorageProviderItem { Id = _.BucketName, Name = _.BucketName, Type = StorageProviderItemType.Folder, ParentReferenceId = parent.Id }); return(items.ToArray()); } else { string bucket; string prefix; GetBucketAndKey(parent.Id, out bucket, out prefix); var request = new ListObjectsV2Request { BucketName = bucket, Prefix = prefix, Delimiter = "/", }; var items = new List <StorageProviderItem>(); ListObjectsV2Response response; do { response = await api.ListObjectsV2Async(request); items.AddRange(response.CommonPrefixes.Select(folderName => new StorageProviderItem { Id = bucket + "/" + folderName, Name = folderName.RemovePrefix(prefix).RemoveTrailingSlash(), Type = StorageProviderItemType.Folder, ParentReferenceId = parent.Id, })); foreach (var o in response.S3Objects) { var normalized = o.Key.RemovePrefix(prefix); if (string.IsNullOrEmpty(normalized) || normalized.EndsWith("/")) // Is Parent Folder (Dummy Item)? => ignore { continue; } items.Add(new StorageProviderItem { Id = bucket + "/" + o.Key, Name = normalized, Type = StorageProviderItemType.File, ParentReferenceId = parent.Id, LastModifiedDateTime = o.LastModified }); } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); return(items.ToArray()); } } }
public ICustomActivityResult Execute() { string Message = string.Empty; var dataTable = new DataTable("List Files", "AWSS3"); try { switch (keyRegion) { case "1": regionEndpoint = RegionEndpoint.USEast2; break; case "2": regionEndpoint = RegionEndpoint.USEast1; break; case "3": regionEndpoint = RegionEndpoint.USWest1; break; case "4": regionEndpoint = RegionEndpoint.USWest2; break; case "5": regionEndpoint = RegionEndpoint.APEast1; break; case "6": regionEndpoint = RegionEndpoint.APSouth1; break; case "7": regionEndpoint = RegionEndpoint.APNortheast2; break; case "8": regionEndpoint = RegionEndpoint.APSoutheast1; break; case "9": regionEndpoint = RegionEndpoint.APSoutheast2; break; case "10": regionEndpoint = RegionEndpoint.APNortheast2; break; case "11": regionEndpoint = RegionEndpoint.CACentral1; break; case "12": regionEndpoint = RegionEndpoint.CNNorth1; break; case "13": regionEndpoint = RegionEndpoint.CNNorthWest1; break; case "14": regionEndpoint = RegionEndpoint.EUCentral1; break; case "15": regionEndpoint = RegionEndpoint.EUWest1; break; case "16": regionEndpoint = RegionEndpoint.EUWest2; break; case "17": regionEndpoint = RegionEndpoint.EUWest3; break; case "18": regionEndpoint = RegionEndpoint.EUNorth1; break; case "19": regionEndpoint = RegionEndpoint.MESouth1; break; case "20": regionEndpoint = RegionEndpoint.SAEast1; break; case "21": regionEndpoint = RegionEndpoint.USGovCloudEast1; break; case "22": regionEndpoint = RegionEndpoint.USGovCloudWest1; break; } client = new AmazonS3Client(AccessKey, SecretKey, regionEndpoint); ListObjectsV2Request request = new ListObjectsV2Request { BucketName = bucketName, MaxKeys = 100 }; ListObjectsV2Response response; do { response = client.ListObjectsV2(request); dataTable.Columns.Add("Id"); dataTable.Columns.Add("FileName"); dataTable.Columns.Add("BucketName"); dataTable.Columns.Add("Size"); dataTable.Columns.Add("ETag"); dataTable.Columns.Add("LastModified"); dataTable.Columns.Add("Owner"); int id = 0; foreach (var item in response.S3Objects) { id++; dataTable.Rows.Add(id, item.Key, item.BucketName, item.Size, item.ETag, item.LastModified, item.Owner); } // Process the response. request.ContinuationToken = response.NextContinuationToken; }while (response.IsTruncated); } catch (AmazonS3Exception amazonS3Exception) { Message = "S3 error occurred. Exception: " + amazonS3Exception.ToString(); return(this.GenerateActivityResult(Message)); } catch (Exception e) { Message = "Exception: " + e.ToString(); Console.ReadKey(); } return(this.GenerateActivityResult(dataTable)); }
public WfResult Run(CancellationToken token) { WfResult result = WfResult.Unknown; //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING])); int runId = 0; Int32.TryParse(_attributes[ETL_RUNID], out runId); int batchId = 0; Int32.TryParse(_attributes[ETL_BATCHID], out batchId); int stepId = 0; Int32.TryParse(_attributes[ETL_STEPID], out stepId); bool setCounterInd = !String.IsNullOrEmpty(_attributes[COUNTER_NAME]); int count = 0; Int32.TryParse(_attributes[COUNT], out count); if (count <= 0) { count = MAX_COUNT; } ControllerCounter counter = new ControllerCounter(_attributes[CONNECTION_STRING], _logger) { BatchId = batchId, StepId = stepId, RunId = runId }; try { //if(!Amazon.Util.ProfileManager.IsProfileKnown(_attributes[PROFILE_NAME])) //{ // Amazon.Util.ProfileManager.RegisterProfile(_attributes[PROFILE_NAME], _attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]); //} //var credentials = new StoredProfileAWSCredentials(_attributes[PROFILE_NAME]); RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(_attributes[REGION_NAME]); AWSCredentials credentials = null; if (!String.IsNullOrEmpty(_attributes[ACCOUNT_NAME])) { credentials = new BasicAWSCredentials(_attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]); } using (var client = new AmazonS3Client(credentials, endpoint)) { //ListBucketsRequest bucketRequest = new ListBucketsRequest(); //ListBucketsResponse backetResponse; //backetResponse = client.ListBuckets(bucketRequest); //var buckets = backetResponse.Buckets; //GetBucketLocationResponse bucketLocationResponse = client.GetBucketLocation(_attributes[CONTAINER_NAME]); ListObjectsV2Request objectRequest = new ListObjectsV2Request { BucketName = _attributes[CONTAINER_NAME], MaxKeys = 100, Prefix = String.IsNullOrEmpty(_attributes[INPUT_PREFIX]) ? null : _attributes[INPUT_PREFIX] }; List <S3Object> objectlist = new List <S3Object>(); ListObjectsV2Response objectResponse; do { objectResponse = client.ListObjectsV2(objectRequest); objectlist.AddRange(objectResponse.S3Objects); objectRequest.ContinuationToken = objectResponse.NextContinuationToken; } while (objectResponse.IsTruncated == true); IEnumerable <S3Object> list = objectlist; if (_attributes[SORT_ORDER].Equals(sortList[0], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <S3Object>().OrderBy(b => b.Key); } if (_attributes[SORT_ORDER].Equals(sortList[1], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <S3Object>().OrderByDescending(b => b.Key); } if (count > 0) { list = list.OfType <S3Object>().Take(count); } Dictionary <string, string> files = new Dictionary <string, string>(); Directory.CreateDirectory(_attributes[OUTPUT_FOLDER]); int i = 0; foreach (var blobItem in list.OfType <S3Object>()) { token.ThrowIfCancellationRequested(); string outputFile = Path.Combine(_attributes[OUTPUT_FOLDER], Path.GetFileName(blobItem.Key)); if (File.Exists(outputFile)) { continue; } GetObjectRequest blobRequest = new GetObjectRequest { BucketName = _attributes[CONTAINER_NAME], Key = blobItem.Key }; using (FileStream outputFileStream = File.Create(outputFile)) using (GetObjectResponse blobResponse = client.GetObject(blobRequest)) using (Stream responseStream = blobResponse.ResponseStream) //using (StreamReader reader = new StreamReader(responseStream)) { responseStream.CopyTo(outputFileStream); //responseBody = reader.ReadToEnd(); } _logger.Debug("downloaded: {File}", outputFile); if (setCounterInd) { files.Add(String.Format("{0}_{1}", _attributes[COUNTER_NAME], i++), outputFile); } } if (setCounterInd) { counter.SetCounters(files); } } } catch (Exception ex) { throw ex; } result = WfResult.Succeeded; return(result); }
private IEnumerable <string> GetParts() { var folder = string.Format("{0}/{1}/raw", Settings.Current.Building.Vendor, Settings.Current.Building.Id); var fileName = string.Empty; foreach (var qd in Settings.Current.Building.SourceQueryDefinitions) { var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database, Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchemaName); if (qd.Persons == null) { continue; } if (string.IsNullOrEmpty(sql)) { continue; } fileName = qd.FileName; break; } if (string.IsNullOrEmpty(fileName)) { fileName = Settings.Current.Building.SourceQueryDefinitions[0].FileName; } using (var client = new AmazonS3Client(Settings.Current.S3AwsAccessKeyId, Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1)) { var request = new ListObjectsV2Request { BucketName = Settings.Current.Bucket, Prefix = string.Format("{0}/{1}/{2}/{3}", folder, chunkId, fileName, fileName) }; ListObjectsV2Response response; do { response = client.ListObjectsV2(request); var partIndexes = new HashSet <string>(); var fn = fileName; foreach (var entry in response.S3Objects) { var end = entry.Key.LastIndexOf(fn, StringComparison.InvariantCultureIgnoreCase) + fn.Length; var key = entry.Key.Replace(entry.Key.Substring(0, end), ""); key = key.Substring(0, key.IndexOf('_')); partIndexes.Add(key); } foreach (var partIndex in partIndexes) { yield return(partIndex); } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); } }
public async Task BulkMoveItems() { // ARRANGE string bucket = sourceBucket; //string bucket = "mhaken-9fc61dff-1893-42cc-a82f-8024960b158e"; Stopwatch sw = new Stopwatch(); int count = 10000; try { sw.Start(); await CreateAndFillBucket(count); sw.Stop(); Debug.Print($"Finished creating and filling bucket {bucket} in {sw.Elapsed}."); sw.Reset(); ListObjectsV2Request list = new ListObjectsV2Request() { BucketName = bucket }; sw.Start(); IEnumerable <S3Object> objects = (await client.ListAllObjectsAsync(list)).SelectMany(x => x.S3Objects); // ACT BulkMoveRequest request = new BulkMoveRequest(objects.Select(x => new CopyObjectRequest() { SourceBucket = x.BucketName, DestinationBucket = x.BucketName, SourceKey = x.Key, DestinationKey = "moved_" + x.Key, StorageClass = S3StorageClass.OneZoneInfrequentAccess, })) { PreferMultipart = true }; BulkCopyResponse response = await client.BulkMoveAsync(request); sw.Stop(); File.WriteAllText("results.txt", $"Successfully moved {count} items in {sw.Elapsed}."); // ASSERT Assert.Equal(objects.Count(), response.SuccessfulOperations.Count); } catch (Exception e) { } finally { await client.EmptyBucketAsync(bucket); await client.DeleteBucketAsync(bucket); } }
public void Fill(AmazonS3Client client, string bucket, string prefix) { //var prefix = $"Lookup/Truven/{fileName}.txt"; var spliter = new StringSplitter(5); var result = new List <string>(); var request = new ListObjectsV2Request { BucketName = bucket, Prefix = prefix }; ListObjectsV2Response response; do { var responseTask = client.ListObjectsV2Async(request); responseTask.Wait(); response = responseTask.Result; foreach (var entry in response.S3Objects) { if (entry.Size > 20) { result.Add(entry.Key); } } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); var getObjectRequest = new GetObjectRequest { BucketName = bucket, Key = result[0] }; var getObject = client.GetObjectAsync(getObjectRequest); getObject.Wait(); using (var responseStream = getObject.Result.ResponseStream) using (var bufferedStream = new BufferedStream(responseStream)) using (var gzipStream = new GZipStream(bufferedStream, CompressionMode.Decompress)) using (var reader = new StreamReader(gzipStream, Encoding.Default)) { string line; while ((line = reader.ReadLine()) != null) { if (!string.IsNullOrEmpty(line)) { spliter.SafeSplit(line, '\t'); var sourceCode = GetStringValue(spliter.Results[0]); if (!_lookup.ContainsKey(sourceCode)) { _lookup.Add(sourceCode, new Dictionary <int, LookupValue>()); } //if (!_lookupIgnoreCase.ContainsKey(sourceCode)) // _lookupIgnoreCase.Add(sourceCode, new HashSet<string>()); //_lookupIgnoreCase[sourceCode].Add(sourceCode); int conceptId = -1; if (int.TryParse(spliter.Results[1], out var cptId)) { conceptId = cptId; } if (!_lookup[sourceCode].ContainsKey(conceptId)) { if (!DateTime.TryParse(spliter.Results[3], out var validStartDate)) { validStartDate = DateTime.MinValue; } if (!DateTime.TryParse(spliter.Results[4], out var validEndDate)) { validEndDate = DateTime.MaxValue; } var lv = new LookupValue { ConceptId = conceptId, SourceCode = sourceCode, Domain = GetStringValue(spliter.Results[2]), ValidStartDate = validStartDate, ValidEndDate = validEndDate, //Ingredients = new HashSet<int>() }; if (spliter.Results.Length > 5) { //lv.SourceVocabularyId = GetStringValue(spliter.Results[5]); lv.SourceConceptId = IsNullOrEmpty(spliter.Results[6]) ? 0 : int.Parse(spliter.Results[6]); //if (!DateTime.TryParse(spliter.Results[7], out var validStartDate2)) // validStartDate2 = DateTime.MinValue; //if (!DateTime.TryParse(spliter.Results[8], out var validEndDate2)) // validEndDate2 = DateTime.MaxValue; //lv.SourceValidStartDate = validStartDate2; //lv.SourceValidEndDate = validEndDate2; } _lookup[sourceCode].Add(conceptId, lv); } if (spliter.Results.Length > 5) { if (!IsNullOrEmpty(spliter.Results[9]) && int.TryParse(spliter.Results[9], out var ingredient)) { if (_lookup[sourceCode][conceptId].Ingredients == null) { _lookup[sourceCode][conceptId].Ingredients = new HashSet <int>(); } _lookup[sourceCode][conceptId].Ingredients.Add(ingredient); } } } } foreach (var v1 in _lookup.Values) { foreach (var v2 in v1.Values) { v2.Ingredients?.TrimExcess(); } } GC.Collect(); } }
public async Task Lock(CancellationToken cancellationToken) { var nowTask = GetUtcTime(cancellationToken); var tasks = new List <Task>(); var lockLifetime = TimeSpan.FromMinutes(3); var listRequest = new ListObjectsV2Request { BucketName = _context.BucketName, Prefix = S3Constants.LockPrefix, }; while (true) { var listResponse = await _context.S3.Invoke((s3, token) => s3.ListObjectsV2Async(listRequest, token), cancellationToken); var threshold = (await nowTask)-lockLifetime; var locked = false; foreach (var s3Object in listResponse.S3Objects) { var key = s3Object.Key; var lockTime = s3Object.LastModified.ToUniversalTime(); if (lockTime < threshold) { var deleteObjectRequest = new DeleteObjectRequest { BucketName = _context.BucketName, Key = key, }; var deleteTask = _context.S3.Invoke((s3, token) => s3.DeleteObjectAsync(deleteObjectRequest, token), cancellationToken); tasks.Add(deleteTask); } else if (key.EndsWith(LockExtension, StringComparison.OrdinalIgnoreCase)) { var keyData = key.Substring(S3Constants.LockPrefix.Length, key.Length - LockExtension.Length - S3Constants.LockPrefix.Length); if (keyData == _lockName) { locked = true; break; } var message = _isLocked ? FormattableString.Invariant($"The lock \"{_lockName}\" was overriden by \"{keyData}\" (time: {lockTime:o}, threshold: {threshold:o}).") : FormattableString.Invariant($"The lock \"{_lockName}\" is prevented by \"{keyData}\" (time: {lockTime:o}, threshold: {threshold:o})."); _isLocked = false; throw new OperationCanceledException(message); } listRequest.StartAfter = key; } if (locked || !listResponse.IsTruncated) { break; } } var request = new PutObjectRequest { BucketName = _context.BucketName, Key = S3Constants.LockPrefix + _lockName + LockExtension, }; var putTask = _context.S3.Invoke((s3, token) => s3.PutObjectAsync(request, token), cancellationToken); tasks.Add(putTask); await Task.WhenAll(tasks); _isLocked = true; }
public static async Task <IActionResult> ListingObjectsAsync(HttpRequest req, TraceWriter log) { // Reference: https://docs.aws.amazon.com/AmazonS3/latest/dev/ListingObjectKeysUsingNetSDK.html var bucketName = Environment.GetEnvironmentVariable("S3BucketName", EnvironmentVariableTarget.Process); var bucketRegion = RegionEndpoint.GetBySystemName(Environment.GetEnvironmentVariable("S3BucketRegion", EnvironmentVariableTarget.Process)); if (awsSecretKey == null) { log.Info($"Fetching AWS secret key for the first time from KeyVault..."); var azureServiceTokenProvider = new AzureServiceTokenProvider(); var keyVaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(azureServiceTokenProvider.KeyVaultTokenCallback)); var secretName = Environment.GetEnvironmentVariable("AmazonS3SecretAccessKeySecretName", EnvironmentVariableTarget.Process); var azureKeyVaultUrl = Environment.GetEnvironmentVariable("AzureKeyVaultUrl", EnvironmentVariableTarget.Process); var secret = await keyVaultClient.GetSecretAsync($"{azureKeyVaultUrl}secrets/{secretName}").ConfigureAwait(false); awsSecretKey = secret.Value; log.Info("[Setting]: Successfully fetched AWS secret key from KeyVault."); } var credentials = new Amazon.Runtime.BasicAWSCredentials(Environment.GetEnvironmentVariable("AwsAccessKey", EnvironmentVariableTarget.Process), awsSecretKey); string s3BucketLastProcessedDateTimeUtcAsString = req.Query["s3BucketLastProcessedDateTimeUtc"]; // GET string requestBody = new StreamReader(req.Body).ReadToEnd(); // POST dynamic data = JsonConvert.DeserializeObject(requestBody); s3BucketLastProcessedDateTimeUtcAsString = s3BucketLastProcessedDateTimeUtcAsString ?? data?.s3BucketLastProcessedDateTimeUtc; if (string.IsNullOrWhiteSpace(s3BucketLastProcessedDateTimeUtcAsString)) { string errorMessage = "A 's3BucketLastProcessedDateTimeUtc' querystring parameter or a request body containing a JSON object with a 's3BucketLastProcessedDateTimeUtc' property was expected but not found."; log.Info(errorMessage); return(new BadRequestObjectResult(errorMessage)); } var s3BucketLastProcessedDateTimeUtc = DateTime.Parse(s3BucketLastProcessedDateTimeUtcAsString); log.Info($"Bucket Name: {bucketName}."); log.Info($"Bucket Region: {bucketRegion}."); log.Info($"S3 Bucket Last Processed DateTimeUtc: {s3BucketLastProcessedDateTimeUtcAsString}."); List <S3Object> filteredObjects = new List <S3Object>(); int totalUnfilteredCount = 0; int currentUnfilteredCount = 0; DateTime newLastProcessedDateTimeUtc = DateTime.UtcNow; IAmazonS3 client = new AmazonS3Client(credentials, bucketRegion); try { ListObjectsV2Request request = new ListObjectsV2Request { BucketName = bucketName }; ListObjectsV2Response response; do { response = await client.ListObjectsV2Async(request); currentUnfilteredCount = response.S3Objects.Count; totalUnfilteredCount += currentUnfilteredCount; log.Info($"Results Count (pre-filtering): {currentUnfilteredCount}."); var currentFilteredObjects = response.S3Objects.FindAll((s3Object) => { // Return objects updated after the last process date and that are not folder records (end with _$folder$ and have 0 size). return(DateTime.Compare(s3Object.LastModified.ToUniversalTime(), s3BucketLastProcessedDateTimeUtc) > 0 & !(s3Object.Key.EndsWith("_$folder$", StringComparison.InvariantCulture) && s3Object.Size == 0)); }); log.Info($"Results Count (post-filtering): {currentFilteredObjects.Count}."); filteredObjects.AddRange(currentFilteredObjects); log.Info($"Next Continuation Token: {response.NextContinuationToken}."); request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); log.Info($"Results Count (total-unfiltered): {totalUnfilteredCount}."); log.Info($"Results Count (total-filtered): {filteredObjects.Count}."); dynamic payload = new System.Dynamic.ExpandoObject(); payload.s3Objects = filteredObjects; payload.newLastProcessedDateTimeUtc = newLastProcessedDateTimeUtc.ToString(); return(new OkObjectResult(JsonConvert.SerializeObject(payload, Formatting.Indented))); } catch (AmazonS3Exception amazonS3Exception) { log.Info($"AmazonS3Exception [ListingObjectsAsync]: {amazonS3Exception.ToString()}."); return(new BadRequestObjectResult("Operation failed (AmazonS3Exception). Check function's log for details.")); } catch (Exception exception) { log.Info($"Exception [ListingObjectsAsync]: {exception.ToString()}."); return(new BadRequestObjectResult("Operation failed. Check function's log for details.")); } }
public async Task <ModuleLocation> ResolveInfoToLocationAsync( ModuleInfo moduleInfo, string bucketName, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError ) { if (bucketName == null) { throw new ArgumentNullException(nameof(bucketName)); } LogInfoVerbose($"... resolving module {moduleInfo}"); StartLogPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}"); var cached = false; try { // check if module can be found in the deployment bucket var result = await FindNewestModuleVersionInBucketAsync(Settings.DeploymentBucketName); // check if the origin bucket needs to be checked if ( allowImport && (Settings.DeploymentBucketName != bucketName) && ( // no version has been found (result.Version == null) // no module version constraint was given; the ultimate floating version || (moduleInfo.Version == null) // the module version constraint is for a pre-release; we always prefer the origin version then || moduleInfo.Version.IsPreRelease() // the module version constraint is floating; we need to check if origin has a newer version || !moduleInfo.Version.Minor.HasValue || !moduleInfo.Version.Patch.HasValue ) ) { var originResult = await FindNewestModuleVersionInBucketAsync(bucketName); // check if module found at origin should be kept instead if ( (originResult.Version != null) && ( (result.Version == null) || (moduleInfo.Version?.IsPreRelease() ?? false) || originResult.Version.IsGreaterThanVersion(result.Version) ) ) { result = originResult; } } // check if a module was found if (result.Version == null) { // could not find a matching version var versionConstraint = (moduleInfo.Version != null) ? $"v{moduleInfo.Version} or later" : "any released version"; if (showError) { if (allowImport) { LogError($"could not find module '{moduleInfo}' ({versionConstraint})"); } else { LogError($"missing module dependency must be imported explicitly '{moduleInfo}' ({versionConstraint})"); } } return(null); } LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}"); return(MakeModuleLocation(result.Origin, result.Manifest)); } finally { StopLogPerformance(cached); } async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionInBucketAsync(string bucketName) { StartLogPerformance($"FindNewestModuleVersionInBucketAsync() for s3://{bucketName}"); try { // enumerate versions in bucket var found = await FindModuleVersionsInBucketAsync(bucketName); if (!found.Any()) { return(Origin : bucketName, Version : null, Manifest : null); } // NOTE (2019-08-12, bjorg): if the module is nested or root, we filter the list of found versions to // only contain versions that meet the module version constraint; for shared modules, we want to // keep the latest version that is compatible with the tool and is equal-or-greater than the // module version constraint. if ( ( (dependencyType == ModuleManifestDependencyType.Root) || (dependencyType == ModuleManifestDependencyType.Nested) ) && (moduleInfo.Version != null) ) { found = found.Where(version => { if (!version.MatchesConstraint(moduleInfo.Version)) { LogInfoVerbose($"... rejected v{version}: does not match version constraint {moduleInfo.Version}"); return(false); } return(true); }).ToList(); } // attempt to identify the newest module version compatible with the tool ModuleManifest manifest = null; var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidateVersion => { var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidateVersion, moduleInfo.Origin); // check if the module version is allowed by the build policy if (!(Settings.BuildPolicy?.Modules?.Allow?.Contains(candidateModuleInfo.ToString()) ?? true)) { LogInfoVerbose($"... rejected v{candidateVersion}: not allowed by build policy"); return(false); } // load module manifest var(candidateManifest, candidateManifestErrorReason) = LoadManifestFromLocationAsync(new ModuleLocation(bucketName, candidateModuleInfo, "<MISSING>")).GetAwaiter().GetResult(); if (candidateManifest == null) { LogInfoVerbose($"... rejected v{candidateVersion}: {candidateManifestErrorReason}"); return(false); } // check if module is compatible with this tool if (!VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(candidateManifest.CoreServicesVersion, Settings.ToolVersion)) { LogInfoVerbose($"... rejected v{candidateVersion}: not compatible with tool version {Settings.ToolVersion}"); return(false); } // keep this manifest manifest = candidateManifest; return(true); }); return(Origin : bucketName, Version : match, Manifest : manifest); } finally { StopLogPerformance(); } } async Task <IEnumerable <VersionInfo> > FindModuleVersionsInBucketAsync(string bucketName) { StartLogPerformance($"FindModuleVersionsInBucketAsync() for s3://{bucketName}"); var cached = false; try { var moduleOrigin = moduleInfo.Origin ?? Settings.DeploymentBucketName; List <VersionInfo> versions = null; string region = null; // check if a cached version exists string cachedManifestVersionsFilePath = null; if (!Settings.ForceRefresh) { var cachedManifestFolder = GetCachedManifestDirectory(bucketName, moduleOrigin, moduleInfo.Namespace, moduleInfo.Name); if (cachedManifestFolder != null) { cachedManifestVersionsFilePath = Path.Combine(cachedManifestFolder, "versions.json"); if ( File.Exists(cachedManifestVersionsFilePath) && (File.GetLastWriteTimeUtc(cachedManifestVersionsFilePath).Add(Settings.CachedManifestListingExpiration) > DateTime.UtcNow) ) { cached = true; var cachedManifestVersions = JsonSerializer.Deserialize <ModuleManifestVersions>(File.ReadAllText(cachedManifestVersionsFilePath), Settings.JsonSerializerOptions); region = cachedManifestVersions.Region; versions = cachedManifestVersions.Versions; } } } // check if data needs to be fetched from S3 bucket if (versions == null) { // get bucket region specific S3 client var s3Client = await GetS3ClientByBucketNameAsync(bucketName); if (s3Client == null) { // nothing to do; GetS3ClientByBucketName already emitted an error return(new List <VersionInfo>()); } // enumerate versions in bucket versions = new List <VersionInfo>(); region = s3Client.Config.RegionEndpoint.SystemName; var request = new ListObjectsV2Request { BucketName = bucketName, Prefix = $"{moduleOrigin}/{moduleInfo.Namespace}/{moduleInfo.Name}/", Delimiter = "/", MaxKeys = 100, RequestPayer = RequestPayer.Requester }; do { try { var response = await s3Client.ListObjectsV2Async(request); versions.AddRange(response.S3Objects .Select(s3Object => s3Object.Key.Substring(request.Prefix.Length)) .Select(found => VersionInfo.Parse(found)) ); request.ContinuationToken = response.NextContinuationToken; } catch (AmazonS3Exception e) when(e.Message == "Access Denied") { // show message that access was denied for this location LogInfoVerbose($"... access denied to {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]"); return(Enumerable.Empty <VersionInfo>()); } } while(request.ContinuationToken != null); // cache module versions listing if (cachedManifestVersionsFilePath != null) { try { File.WriteAllText(cachedManifestVersionsFilePath, JsonSerializer.Serialize(new ModuleManifestVersions { Region = region, Versions = versions }, Settings.JsonSerializerOptions)); } catch { // nothing to do } } } // filter list down to matching versions versions = versions.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)).ToList(); LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{region}]"); return(versions); } finally { StopLogPerformance(cached); } } ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest) => new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum); }
public async Task <ListObjectsV2Response> ListObjectsV2Async(ListObjectsV2Request request) => await ExecuteS3Request(() => _s3Client.ListObjectsV2Async(request), request.BucketName);
/// <summary> /// This routine will retireve all objects within the specified S3 Bucket /// Any objects which have a modified date older than our target will be removed /// </summary> /// <param name="modifiedDate"></param> /// <returns></returns> private static async Task <List <S3Object> > ListObjects(DateTime modifiedDate) { try { List <S3Object> s3Objects = new List <S3Object>(); ListObjectsV2Request request = new ListObjectsV2Request() { BucketName = bucketName }; ListObjectsV2Response response; Console.WriteLine("Connecting to S3..."); do { response = await client.ListObjectsV2Async(request); foreach (S3Object entry in response.S3Objects) { if (entry.LastModified.IsAfter(modifiedDate)) { var listResponse = client.ListVersionsAsync(new ListVersionsRequest { BucketName = bucketName, Prefix = entry.Key }); S3ObjectVersion deleteMarkerVersion = listResponse.Result.Versions.FirstOrDefault(x => x.IsLatest); if (deleteMarkerVersion != null) { await client.DeleteObjectAsync(new DeleteObjectRequest { BucketName = bucketName, Key = entry.Key, VersionId = deleteMarkerVersion.VersionId }); } s3Objects.Add(entry); } } Console.WriteLine("Next Continuation Token: {0}", response.NextContinuationToken); request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); return(s3Objects); } catch (AmazonS3Exception e) { Console.WriteLine("Error encountered on server. Message:'{0}' when writing an object", e.Message); return(null); } catch (Exception e) { Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message); return(null); } }
async Task <ListObjectsV2Response> IS3Client.ListObjectsV2Async(ListObjectsV2Request request) => await _amazonS3.ListObjectsV2Async(request);
private async Task GetObjectList(string BucketName, string Prefix) { this.StartSpinnerFrame(); this.BucketName = BucketName; this.FolderKey = Prefix; this.NavigationPanel.Children.Clear(); this.NavigationPanel.Children.Add(this.GetHyperLinkLabel(BucketName, "", RefreshFolder)); if (this.FolderKey != "") { string[] FolderKeys = this.FolderKey.Split('/'); string FolderKeyOrigin = ""; foreach (string Key in FolderKeys) { FolderKeyOrigin += Key + "/"; if (Key != "") { this.NavigationPanel.Children.Add(new Label { Content = " > " }); int iChild = this.NavigationPanel.Children.Add(this.GetHyperLinkLabel(FolderKey, FolderKeyOrigin, RefreshFolder)); } } } var request = new ListObjectsV2Request { BucketName = BucketName , Prefix = Prefix , Delimiter = "/" }; ObjectList.Items.Clear(); ListObjectsV2Response response; do { response = await this.s3ClientHandler.ListObjectsV2Async(request); foreach (string commonPrefix in response.CommonPrefixes) { var s3Objects = new S3Objects { Image = ResourceHelper.GetBitmapFrameForResourceImage(S3Browser.Properties.Resources.blue_folder) , Type = "Folder" , BucketName = BucketName , Key = commonPrefix , Name = this.FolderKey.Length > 0 ? commonPrefix.Replace(this.FolderKey, "") : commonPrefix }; ListViewItem item = new ListViewItem { Content = s3Objects }; _ = this.ObjectList.Items.Add(item); } foreach (S3Object s3Object in response.S3Objects) { var s3Objects = new S3Objects { Image = ResourceHelper.GetBitmapFrameForResourceImage(S3Browser.Properties.Resources.document) , Type = "Object" , BucketName = s3Object.BucketName , Key = s3Object.Key , Name = this.FolderKey.Length > 0 ? s3Object.Key.Replace(this.FolderKey, "") : s3Object.Key , LastModified = s3Object.LastModified , Owner = s3Object.Owner , StorageClass = s3Object.StorageClass }; s3Objects.SetSize(s3Object.Size); _ = this.ObjectList.Items.Add(new ListViewItem { Content = s3Objects }); } request.ContinuationToken = response.NextContinuationToken; this.EndSpinnerFrame(); } while (response.IsTruncated); }
public List <FileListModel> GetDirectoryList(string directory, string accessToken) { var config = new AmazonS3Config { RegionEndpoint = RegionEndpoint.USWest1, // MUST set this before setting ServiceURL and it should match the `MINIO_REGION` enviroment variable. ForcePathStyle = true // MUST be true to work correctly with Minio server }; string region = accessToken.Split("-")[2]; if (region == "USWest1") { config.RegionEndpoint = RegionEndpoint.USWest1; } else if (region == "USWest2") { config.RegionEndpoint = RegionEndpoint.USWest2; } else if (region == "APNortheast1") { config.RegionEndpoint = RegionEndpoint.APNortheast1; } else if (region == "APNortheast2") { config.RegionEndpoint = RegionEndpoint.APNortheast2; } else if (region == "APSouth1") { config.RegionEndpoint = RegionEndpoint.APSouth1; } else if (region == "APSoutheast1") { config.RegionEndpoint = RegionEndpoint.APSoutheast1; } else if (region == "APSoutheast2") { config.RegionEndpoint = RegionEndpoint.APSoutheast2; } else if (region == "CACentral1") { config.RegionEndpoint = RegionEndpoint.CACentral1; } else if (region == "CNNorth1") { config.RegionEndpoint = RegionEndpoint.CNNorth1; } else if (region == "CNNorthWest1") { config.RegionEndpoint = RegionEndpoint.CNNorthWest1; } else if (region == "EUCentral1") { config.RegionEndpoint = RegionEndpoint.EUCentral1; } else if (region == "EUWest2") { config.RegionEndpoint = RegionEndpoint.EUWest2; } else if (region == "EUWest3") { config.RegionEndpoint = RegionEndpoint.EUWest3; } else if (region == "SAEast1") { config.RegionEndpoint = RegionEndpoint.SAEast1; } else if (region == "USEast1") { config.RegionEndpoint = RegionEndpoint.USEast1; } else if (region == "USEast2") { config.RegionEndpoint = RegionEndpoint.USEast2; } else if (region == "EUWest1") { config.RegionEndpoint = RegionEndpoint.EUWest1; } else if (region == "USGovCloudWest1") { config.RegionEndpoint = RegionEndpoint.USGovCloudWest1; } //var amazonS3Client = new AmazonS3Client("AKIAIZ7ZNECIUB4MEKRQ", "g2OZECVv8kRsbixfiP52xnKhHo/FSny5hm0qmb4q", config); var amazonS3Client = new AmazonS3Client(accessToken.Split("-")[0], accessToken.Split("-")[1], config); List <FileListModel> list = new List <FileListModel>(); if (String.IsNullOrEmpty(directory)) { var listBucketResponse = (amazonS3Client.ListBucketsAsync()).Result; foreach (var item in listBucketResponse.Buckets) { list.Add(new FileListModel() { CloudId = item.BucketName, Directory = item.BucketName }); } } else { string bucket = Regex.Split(directory, "/")[0]; string innerDirectory = ""; string[] directoryArray = directory.Split("/"); ListObjectsV2Request request = new ListObjectsV2Request { BucketName = bucket, Delimiter = @"/", }; if (directoryArray.Length > 1) { for (int i = 1; i < directoryArray.Length - 1; i++) { innerDirectory = directoryArray[i] + "/"; } request.Prefix = innerDirectory; } else { } var bucketDirectory = (amazonS3Client.ListObjectsV2Async(request)).Result; foreach (var item in bucketDirectory.CommonPrefixes) { list.Add(new FileListModel() { CloudId = bucket + "/" + item, Directory = bucket + "/" + item }); } } return(list); }
/// <inheritdoc/> public Task <ListObjectsV2Response> ListObjectsV2Async(ListObjectsV2Request request, CancellationToken cancellationToken = default(CancellationToken)) { throw new NotImplementedException(); }
public void ListObjectsV2Async(ListObjectsV2Request request, AmazonServiceCallback <ListObjectsV2Request, ListObjectsV2Response> callback, AsyncOptions options = null) { throw new System.NotImplementedException(); }
protected void btnUpload_Click(object sender, EventArgs e) { /* * First upload image in bucket * detect face while uploading image into bucket * compare face in face collection * get list of image from bucket * store uploaded image in face collection * * compare face in face collection while uploading * * * */ try { image.Visible = false; lblMessage.Text = ""; string imagefile = string.Empty; if (fuImage.HasFile) { string imagename = System.IO.Path.GetFullPath(fuImage.PostedFile.FileName); //Label1.Text = imagename; string ext = System.IO.Path.GetExtension(fuImage.FileName); // Label2.Text = ext; // imagefile = Server.MapPath("~/Images/" + imagename); imagefile = imagename; if (ext == ".jpg" | ext == ".png") { //fuImage.SaveAs(imagefile); Stream st = fuImage.PostedFile.InputStream; IAmazonRekognition rekoClient = new AmazonRekognitionClient(Amazon.RegionEndpoint.USEast1); string name = Path.GetFileName(fuImage.PostedFile.FileName); name = @"C:\Blazar\CompareImage\" + fuImage.FileName; string myBucketName = "blazarstorage"; //your s3 bucket name goes here string s3DirectoryName = ""; string s3FileName = @name; string fileName = fuImage.FileName; /* * StoreFaceInCollection(@name, rekoClient, myBucketName); * * ImageRecognition imageRecog = new ImageRecognition(); * IdentifyFaces(name, fuImage.PostedFile.FileName.ToString(), myBucketName); * */ //STEP----1 //validate the image is a face or not--step 1 //bool isFaceImage = DetectFaces(fileName); //bool isFaceImage = DetectFaces(name); bool isFaceImage = true; if (isFaceImage == true) { /* * we can compare image if those are inside buckets. * For comparing images , we have to follow below steps * store it in bucket * add to it face collection * compare images * */ // bool isUpload; //upload image into bucket isUpload = sendMyFileToS3(st, myBucketName, s3DirectoryName, s3FileName, fileName); if (isUpload == true) { //store image in a face collection StoreFaceInCollection(@name, rekoClient, myBucketName); // AmazonUploader myUploader = new AmazonUploader(); // bool b = myUploader.IsFilExist("AKIAIPXX2OIGXNKENL6Q", "oQQn3l4ll5/J/OY2RoZG4EV4RZtv8EsD114MrRnR"); //validate the existance of the image in the bucket //for that we have to get all the face from face collection try { AmazonS3Client s3Client = new AmazonS3Client(Amazon.RegionEndpoint.USEast1); ListObjectsV2Request reqGetObjFromBucket = new ListObjectsV2Request { BucketName = myBucketName, MaxKeys = 1000 }; ListObjectsV2Response resGetObjFromBucket; do { resGetObjFromBucket = s3Client.ListObjectsV2(reqGetObjFromBucket); foreach (Amazon.S3.Model.S3Object entry in resGetObjFromBucket.S3Objects) { //if (DetectFaces(entry.Key))//validat the image content //{ // if (s3FileName != entry.Key)//input image should not compare if (fileName != entry.Key) { var response = rekoClient.CompareFaces(new CompareFacesRequest { SimilarityThreshold = 90, SourceImage = new Amazon.Rekognition.Model.Image { S3Object = new Amazon.Rekognition.Model.S3Object { Bucket = myBucketName, // Name="" // Name = s3FileName Name = fileName } }, TargetImage = new Amazon.Rekognition.Model.Image { S3Object = new Amazon.Rekognition.Model.S3Object { Bucket = myBucketName, // Name="" Name = entry.Key } } }); if (response.FaceMatches.Count > 0) { image.Visible = true; dupImage.Src = name; existingImage.Src = "https://s3.amazonaws.com/blazarstorage/" + entry.Key;; lblMessage.Text = "You are trying to upload the image " + s3FileName + " which is matching with " + entry.Key; lblMessage.ForeColor = System.Drawing.Color.Green; IAmazonS3 s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast1); s3.DeleteObject(myBucketName, fileName); return; } } // } } } while (resGetObjFromBucket.IsTruncated == true); } catch (Exception ex) { IAmazonS3 s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast1); s3.DeleteObject(myBucketName, fileName); lblMessage.Text = ex.Message.ToString(); lblMessage.ForeColor = System.Drawing.Color.Red; } // List< Face > lstOfFaces= GetListOfFaceInFaceCollection(); //if (lstOfFaces.Count >= 1) //{ // for(int i=0;i<lstOfFaces.Count;i++) // { // GetObjectRequest objReq = new GetObjectRequest(); // GetObjectResponse res = new GetObjectResponse(); // if (response.FaceMatches.Count > 0) // { // IAmazonS3 s3 = new AmazonS3Client(); // s3.DeleteObject(myBucketName, s3FileName); // } // } //} } else { lblMessage.Text = "Please upload again!"; } //else //{ // //upload image into bucket--step 3 // isUpload = sendMyFileToS3(st, myBucketName, s3DirectoryName, s3FileName); // if (isUpload == true) // { // //store image in a face collection // StoreFaceInCollection(@name, rekoClient, myBucketName); // lblMessage.Text = "successfully uploaded"; // Response.Write("successfully uploaded"); // } // else // lblMessage.Text = "error"; //} } else { lblMessage.Text = "Please upload a valid image!!!"; } } } } catch (Exception ex) { lblMessage.Text = ex.Message.ToString(); } }
// Please set the following connection strings in app.config for this WebJob to run: // AzureWebJobsDashboard and AzureWebJobsStorage private static void Main() { #region Parse Configuration Settings AwsSettings awsSettings = new AwsSettings(); string AWS_SETTINGS = CloudConfigurationManager.GetSetting("AWS_SETTINGS"); awsSettings = JsonConvert.DeserializeObject <AwsSettings>(AWS_SETTINGS); azureStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureStorageConnectionString")); azureStorageContainerName = CloudConfigurationManager.GetSetting("AzureStorageContainerName"); string runMode = CloudConfigurationManager.GetSetting("runMode"); if (runMode == "INTERACTIVE") { interactiveMode = true; } #endregion Console.WriteLine("CopyS3toBlob: {0}.", awsSettings.Description); azureClient = azureStorageAccount.CreateCloudBlobClient(); azureClient.GetContainerReference(azureStorageContainerName).CreateIfNotExists(); // For each account in AwsSettings foreach (var account in awsSettings.Accounts) { ConsoleColor currentColor = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine(@"Processing AWS Account {0}", account); Console.ForegroundColor = currentColor; // For each bucket we need to inspect for this account foreach (var bucket in account.Buckets) { Console.WriteLine(@"Processing Bucket {0}", bucket.Name); count = 0; files = new Dictionary <string, S3Object>(); Amazon.RegionEndpoint region = GetAwsRegionFromString(bucket.Region); using (client = new AmazonS3Client(account.AccessKey, account.SecretAccessKey, region)) { try { ListObjectsV2Request request = new ListObjectsV2Request { BucketName = bucket.Name, MaxKeys = 10, Prefix = account.AccountId }; ListObjectsV2Response response; do { response = client.ListObjectsV2(request); // Process response. foreach (S3Object entry in response.S3Objects) { if (FileIsRequired(entry.Key, bucket.Patterns)) { count++; //Console.WriteLine("key = {0} size = {1}, Date={2}", entry.Key, entry.Size, entry.LastModified); S3Object theFile = new S3Object(); theFile.BucketName = entry.BucketName; theFile.ETag = entry.ETag; theFile.LastModified = entry.LastModified; theFile.Owner = entry.Owner; theFile.Size = entry.Size; theFile.StorageClass = entry.StorageClass; files.Add(entry.Key, theFile); } } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated == true); } catch (AmazonS3Exception amazonS3Exception) { if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity"))) { Console.WriteLine("Check the provided AWS Credentials."); Console.WriteLine( "To sign up for service, go to http://aws.amazon.com/s3"); } else { Console.WriteLine( "Error occurred. Message:'{0}' when listing objects", amazonS3Exception.Message); } throw amazonS3Exception; } } ConsoleColor savedColour = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Bucket{0} contains {1} files of interest:", bucket.Name, count); Console.ForegroundColor = savedColour; BucketBlobPackage package = new BucketBlobPackage(); package.AccountId = account.AccountId; package.AccessKey = account.AccessKey; package.SecretAccessKey = account.SecretAccessKey; package.BucketInfo = bucket; package.FileInfo = files; WriteToBlobStorage(package); } } Console.WriteLine("CopyS2toBlob job has completed sucessfully"); if (interactiveMode) { Console.ReadKey(); } }
public async Task <ListObjectsV2Response> ListObjectsV2Async(ListObjectsV2Request request) => await _s3Client.ListObjectsV2Async(request);
/// <summary> /// /// <para>DeleteFolder:</para> /// /// <para>Deletes a folder from File Service, caller thread will be blocked before it is done</para> /// /// <para>Check <seealso cref="IBFileServiceInterface.DeleteFile"/> for detailed documentation</para> /// /// </summary> public bool DeleteFolder( string _BucketName, string _Folder, Action <string> _ErrorMessageAction = null) { if (S3Client == null) { _ErrorMessageAction?.Invoke("BFileServiceAWS->DeleteFolder: S3Client is null."); return(false); } var PrefixedObjects = new List <KeyVersion>(); try { ListObjectsV2Request ListRequest = new ListObjectsV2Request() { BucketName = _BucketName, Prefix = _Folder }; using (var ListObjectsTask = S3Client.ListObjectsV2Async(ListRequest)) { ListObjectsTask.Wait(); foreach (var FileObject in ListObjectsTask.Result.S3Objects) { if (FileObject != null) { PrefixedObjects.Add(new KeyVersion() { Key = FileObject.Key }); } } } } catch (Exception e) { PrefixedObjects = null; _ErrorMessageAction?.Invoke("BFileServiceAWS->DeleteFolder-ListPrefixed: " + e.Message + ", Trace: " + e.StackTrace); return(false); } DeleteObjectsRequest Request = new DeleteObjectsRequest { BucketName = _BucketName, Objects = PrefixedObjects }; try { using (var CreatedTask = S3Client.DeleteObjectsAsync(Request)) { CreatedTask.Wait(); } } catch (Exception e) { _ErrorMessageAction?.Invoke("BFileServiceAWS->DeleteFolder: " + e.Message + ", Trace: " + e.StackTrace); return(false); } return(true); }
public async Task <ListObjectsV2Response> GetListAsync(ListObjectsV2Request request) { return(await _s3.ListObjectsV2Async(request)); }
public async Task <IActionResult> GetListingAsync(ListObjectsV2Request request) { var response = await _fileServices.GetListAsync(request); return(Ok(response)); }
static async Task <List <KeyVersion> > ListingObjectsAsync() { List <KeyVersion> keyVersion = new List <KeyVersion>(); bool isContinue = false; try { ListObjectsV2Request request = new ListObjectsV2Request { BucketName = bucketName, MaxKeys = 20, Delimiter = "/", Prefix = prefix // "private/events/" + eventCode +"/sessions/" }; ListObjectsV2Response response; do { response = await s3Client.ListObjectsV2Async(request); List <KeyVersion> tempkeyVersion = new List <KeyVersion>(); int counter = 1; // Process the response. foreach (S3Object entry in response.S3Objects) { Console.WriteLine("id = {0} key = {1} size = {2}", counter, entry.Key, entry.Size); tempkeyVersion.Add(new KeyVersion() { Key = entry.Key, VersionId = counter.ToString() }); counter++; } //Console.WriteLine("Next Continuation Token: {0}", response.NextContinuationToken); request.ContinuationToken = response.NextContinuationToken; //take comma seperated list ok item and put to List //then ask for new list or delete isContinue = false; Console.WriteLine("1. Enter 1 for entering comma seperated list of objects to be deleted"); Console.WriteLine("2. Fetch new list of assets"); string input = Console.ReadLine(); try { int action = Convert.ToInt32(input); switch (action) { case 1: Console.WriteLine("Enter asset id corresponding to asset item"); input = Console.ReadLine(); var tagIds = new List <string>(input.Split(',')); foreach (var item in tagIds) { var query = tempkeyVersion.Where(x => x.VersionId == item).FirstOrDefault(); keyVersion.Add(new KeyVersion { Key = query.Key, VersionId = "1" }); } break; case 2: isContinue = true; break; default: throw new Exception(); break; } } catch (Exception ex) { Console.WriteLine("No such command."); } } while (response.IsTruncated && isContinue); } catch (AmazonS3Exception amazonS3Exception) { Console.WriteLine("S3 error occurred. Exception: " + amazonS3Exception.ToString()); Console.ReadKey(); } catch (Exception e) { Console.WriteLine("Exception: " + e.ToString()); Console.ReadKey(); } return(keyVersion); }
public async Task <int> DeleteFilesAsync(string searchPattern = null, CancellationToken cancellationToken = new CancellationToken()) { var criteria = GetRequestCriteria(searchPattern); int count = 0; const int PAGE_SIZE = 100; var listRequest = new ListObjectsV2Request { BucketName = _bucket, Prefix = criteria.Prefix, MaxKeys = PAGE_SIZE }; var deleteRequest = new DeleteObjectsRequest { BucketName = _bucket }; var errors = new List <DeleteError>(); ListObjectsV2Response listResponse; do { listResponse = await _client.ListObjectsV2Async(listRequest, cancellationToken).AnyContext(); listRequest.ContinuationToken = listResponse.NextContinuationToken; var keys = listResponse.S3Objects.MatchesPattern(criteria.Pattern).Select(o => new KeyVersion { Key = o.Key }).ToArray(); if (keys.Length == 0) { continue; } deleteRequest.Objects.AddRange(keys); var deleteResponse = await _client.DeleteObjectsAsync(deleteRequest, cancellationToken).AnyContext(); if (deleteResponse.DeleteErrors.Count > 0) { // retry 1 time, continue on. var deleteRetryRequest = new DeleteObjectsRequest { BucketName = _bucket }; deleteRetryRequest.Objects.AddRange(deleteResponse.DeleteErrors.Select(e => new KeyVersion { Key = e.Key })); var deleteRetryResponse = await _client.DeleteObjectsAsync(deleteRetryRequest, cancellationToken).AnyContext(); if (deleteRetryResponse.DeleteErrors.Count > 0) { errors.AddRange(deleteRetryResponse.DeleteErrors); } } count += deleteResponse.DeletedObjects.Count; deleteRequest.Objects.Clear(); } while (listResponse.IsTruncated && !cancellationToken.IsCancellationRequested); if (errors.Count > 0) { int more = errors.Count > 20 ? errors.Count - 20 : 0; throw new Exception($"Unable to delete all S3 entries \"{String.Join(",", errors.Take(20).Select(e => e.Key))}\"{(more > 0 ? $" plus {more} more" : "")}."); } return(count); }
private async Task<IEnumerable<SourceImage>> ListImagesAsync( ServiceProvider provider) { var entries = await this.GetImageMapAsync(provider.GetService<BucketMonitorContext>()); var results = new List<SourceImage>(); ListObjectsV2Request request = new ListObjectsV2Request { BucketName = this.BucketName }; var console = new ConsoleString(); console.Update($"Loading Objects..."); int cacheAdditions = 0; int count = 0; ListObjectsV2Response response; do { using (var scoped = provider.CreateScope()) { var dbContext = scoped.ServiceProvider.GetService<BucketMonitorContext>(); dbContext.ChangeTracker.AutoDetectChangesEnabled = false; var tracked = await this.GetTrackedBucket(dbContext); response = await this.Client.ListObjectsV2Async(request); var images = new List<SourceImage>(); var toAdd = new List<ImageEntry>(); var filtered = response.S3Objects .Where(x => x.Key.Length <= MAX_PATH_LENGTH); foreach (var obj in filtered) { var image = this.ProcessObject( tracked: tracked, obj: obj, entries: entries, toAdd: toAdd); images.Add(image); } if (toAdd.Count() > 0) { dbContext.Image.AddRange(toAdd); await dbContext.SaveChangesAsync(); cacheAdditions += toAdd.Count(); } results.AddRange(images); request.ContinuationToken = response.NextContinuationToken; count++; if (cacheAdditions > 0) { console.Update($"Loading Objects: {results.Count()}, CacheAdditions={cacheAdditions}"); } else { console.Update($"Loading Objects: {results.Count()}"); } } } while (response.IsTruncated); console.Update($"Loaded {results.Count()} Objects"); this.Logger.LogDebug($"Queried {results.Count()} Objects"); return results; }
public async Task <ModuleLocation> ResolveInfoToLocationAsync(ModuleInfo moduleInfo, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError, bool allowCaching = false) { LogInfoVerbose($"... resolving module {moduleInfo}"); var stopwatch = Stopwatch.StartNew(); var cached = false; try { // check if a cached manifest matches var cachedDirectory = Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo)); if (allowCaching && Settings.AllowCaching && Directory.Exists(cachedDirectory)) { var foundCached = Directory.GetFiles(cachedDirectory) .Select(found => VersionInfo.Parse(Path.GetFileName(found))) .Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)); // NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to // only contain versions that meet the module version constraint; for shared modules, we want to // keep the latest version that is compatible with the tool and is equal-or-greater than the // module version constraint. if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null)) { foundCached = foundCached.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList(); } // attempt to identify the newest module version compatible with the tool ModuleManifest manifest = null; var match = VersionInfo.FindLatestMatchingVersion(foundCached, moduleInfo.Version, candidate => { var candidateManifestText = File.ReadAllText(Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo), candidate.ToString())); manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText); // check if module is compatible with this tool return(manifest.CoreServicesVersion.IsCoreServicesCompatible(Settings.ToolVersion)); }); if (manifest != null) { cached = true; // TODO (2019-10-08, bjorg): what source bucket name should be used for cached manifests? return(MakeModuleLocation(Settings.DeploymentBucketName, manifest)); } } // check if module can be found in the deployment bucket var result = await FindNewestModuleVersionAsync(Settings.DeploymentBucketName); // check if the origin bucket needs to be checked if ( allowImport && (Settings.DeploymentBucketName != moduleInfo.Origin) && ( // no version has been found (result.Version == null) // no module version constraint was given; the ultimate floating version || (moduleInfo.Version == null) // the module version constraint is for a pre-release; we always prefer the origin version then || moduleInfo.Version.IsPreRelease // the module version constraint is floating; we need to check if origin has a newer version || moduleInfo.Version.HasFloatingConstraints ) ) { var originResult = await FindNewestModuleVersionAsync(moduleInfo.Origin); // check if module found at origin should be kept instead if ( (originResult.Version != null) && ( (result.Version == null) || (moduleInfo.Version?.IsPreRelease ?? false) || originResult.Version.IsGreaterThanVersion(result.Version) ) ) { result = originResult; } } // check if a module was found if (result.Version == null) { // could not find a matching version var versionConstraint = (moduleInfo.Version != null) ? $"v{moduleInfo.Version} or later" : "any version"; if (showError) { LogError($"could not find module '{moduleInfo}' ({versionConstraint})"); } return(null); } LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}"); // cache found version Directory.CreateDirectory(cachedDirectory); await File.WriteAllTextAsync(Path.Combine(cachedDirectory, result.Version.ToString()), JsonConvert.SerializeObject(result.Manifest)); return(MakeModuleLocation(result.Origin, result.Manifest)); } finally { LogInfoPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}", stopwatch.Elapsed, cached); } async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionAsync(string bucketName) { // enumerate versions in bucket var found = await FindModuleVersionsAsync(bucketName); if (!found.Any()) { return(Origin : bucketName, Version : null, Manifest : null); } // NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to // only contain versions that meet the module version constraint; for shared modules, we want to // keep the latest version that is compatible with the tool and is equal-or-greater than the // module version constraint. if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null)) { found = found.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList(); } // attempt to identify the newest module version compatible with the tool ModuleManifest manifest = null; var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidate => { var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidate, moduleInfo.Origin); var candidateManifestText = GetS3ObjectContentsAsync(bucketName, candidateModuleInfo.VersionPath).Result; manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText); // check if module is compatible with this tool return(manifest.CoreServicesVersion.IsCoreServicesCompatible(Settings.ToolVersion)); }); return(Origin : bucketName, Version : match, Manifest : manifest); } async Task <IEnumerable <VersionInfo> > FindModuleVersionsAsync(string bucketName) { // get bucket region specific S3 client var s3Client = await GetS3ClientByBucketNameAsync(bucketName); if (s3Client == null) { // nothing to do; GetS3ClientByBucketName already emitted an error return(new List <VersionInfo>()); } // enumerate versions in bucket var versions = new List <VersionInfo>(); var request = new ListObjectsV2Request { BucketName = bucketName, Prefix = $"{moduleInfo.Origin ?? Settings.DeploymentBucketName}/{moduleInfo.Namespace}/{moduleInfo.Name}/", Delimiter = "/", MaxKeys = 100, RequestPayer = RequestPayer.Requester }; do { try { var response = await s3Client.ListObjectsV2Async(request); versions.AddRange(response.S3Objects .Select(s3Object => s3Object.Key.Substring(request.Prefix.Length)) .Select(found => VersionInfo.Parse(found)) .Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)) ); request.ContinuationToken = response.NextContinuationToken; } catch (AmazonS3Exception e) when(e.Message == "Access Denied") { break; } } while(request.ContinuationToken != null); LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]"); return(versions); } ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest) => new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum); }
public async Task <IActionResult> OnPostAsync(int?id, int?coid) { if (id == null) { return(Redirect(Url.Content("~/Forbidden")));; } Contest = await _context.Contest.FindAsync(id); Settings = await _context.Settings.FirstOrDefaultAsync(m => m.CONTESTID == id); Questions = await _context.Questions.FirstOrDefaultAsync(m => m.CONTESTID == id); Mails = await _context.Mails.FirstOrDefaultAsync(m => m.CONTESTID == id); Company = await _context.Company.FirstOrDefaultAsync(m => m.Id == coid); AnswersList = await _context.Answers.Where(x => x.CONTESTID == Contest.ID).ToListAsync(); PlayerList = await _context.Player.Where(x => x.CONTESTID == Contest.ID).ToListAsync(); if (Contest != null) { await DeleteFile(); ListObjectsV2Request request = new ListObjectsV2Request { BucketName = _bucketName, Prefix = _context.Company.FirstOrDefaultAsync(m => m.Id == _userManager.GetUserAsync(User).Result.CompanyId).Result.CompanyName + "/" + Contest.NAME + "/" }; ListObjectsV2Response response; do { response = await _s3Client.ListObjectsV2Async(request); // Process the response. foreach (S3Object entry in response.S3Objects) { await FileHandler.DeleteFileS3(entry.Key); } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); foreach (var item in AnswersList) { _context.Answers.Remove(item); } foreach (var item in PlayerList) { _context.Player.Remove(item); } _context.Contest.Remove(Contest); _context.Settings.Remove(Settings); _context.Questions.Remove(Questions); _context.Mails.Remove(Mails); await _context.SaveChangesAsync(); } return(Redirect(Url.Content("~/Contests/") + Company.Id)); }