/// <summary>
/// Gets the objects in folder without recursion. i.e. will get the list of files
/// and folders in the folder but not the contents of the subfolders. Subfolders
/// will not have the ModifiedDate prop filled in as Amazon doesn't provide it in
/// this context.
/// </summary>
/// <param name="assetStorageProvider">The asset storage Provider.</param>
/// <param name="asset">The asset.</param>
/// <returns></returns>
public override List <Asset> ListObjectsInFolder(AssetStorageProvider assetStorageProvider, Asset asset)
{
string rootFolder = FixRootFolder(GetAttributeValue(assetStorageProvider, AttributeKeys.RootFolder));
string bucketName = GetAttributeValue(assetStorageProvider, AttributeKeys.Bucket);
asset.Key = FixKey(asset, rootFolder);
HasRequirementsFolder(asset);
try
{
AmazonS3Client client = GetAmazonS3Client(assetStorageProvider);
ListObjectsV2Request request = new ListObjectsV2Request();
request.BucketName = bucketName;
request.Prefix = asset.Key == "/" ? rootFolder : asset.Key;
request.Delimiter = "/";
var assets = new List <Asset>();
var subFolders = new HashSet <string>();
ListObjectsV2Response response;
// S3 will only return 1,000 keys per response and sets IsTruncated = true, the do-while loop will run and fetch keys until IsTruncated = false.
do
{
response = client.ListObjectsV2(request);
foreach (S3Object s3Object in response.S3Objects)
{
if (s3Object.Key == null)
{
continue;
}
var responseAsset = CreateAssetFromS3Object(assetStorageProvider, s3Object, client.Config.RegionEndpoint.SystemName);
assets.Add(responseAsset);
}
// After setting the delimiter S3 will filter out any prefixes below that in response.S3Objects.
// So we need to inspect response.CommonPrefixes to get the prefixes inside the folder.
foreach (string subFolder in response.CommonPrefixes)
{
if (subFolder.IsNotNullOrWhiteSpace())
{
subFolders.Add(subFolder);
}
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
// Add the subfolders to the asset collection
foreach (string subFolder in subFolders)
{
var subFolderAsset = CreateAssetFromCommonPrefix(subFolder, client.Config.RegionEndpoint.SystemName, bucketName);
assets.Add(subFolderAsset);
}
return(assets.OrderBy(a => a.Key, StringComparer.OrdinalIgnoreCase).ToList());
}
catch (Exception ex)
{
ExceptionLogService.LogException(ex);
throw;
}
}
private async Task ShowFailedKinesisProcessingLogs(Settings settings, string keyPrefix)
{
// populate information about the deployment tier
await PopulateDeploymentTierSettingsAsync(settings);
if (settings.LoggingBucketName == null)
{
LogError("Deployment tier does not a have logging bucket");
return;
}
// list all objects under the specified key
var request = new ListObjectsV2Request {
BucketName = settings.LoggingBucketName,
Prefix = keyPrefix,
MaxKeys = 100
};
do
{
var response = await settings.S3Client.ListObjectsV2Async(request);
foreach (var s3Object in response.S3Objects)
{
Console.WriteLine($"{Settings.OutputColor}Key:{Settings.ResetColor}: {s3Object.Key}");
try {
var records = await GetS3ObjectContentsAsync(s3Object.Key);
foreach (var record in records)
{
Console.WriteLine($"{Settings.InfoColor}ArrivalTime:{Settings.ResetColor} {DateTimeOffset.FromUnixTimeMilliseconds(record.arrivalTimestamp)}");
Console.WriteLine($"{Settings.InfoColor}AttemptEndingTimestamp:{Settings.ResetColor} {DateTimeOffset.FromUnixTimeMilliseconds(record.attemptEndingTimestamp)}");
Console.WriteLine($"{Settings.InfoColor}AttemptsMade:{Settings.ResetColor} {record.attemptsMade}");
Console.WriteLine($"{Settings.InfoColor}ErrorCode:{Settings.ResetColor} {record.errorCode}");
Console.WriteLine($"{Settings.InfoColor}ErrorMessage:{Settings.ResetColor} {record.errorMessage}");
Console.WriteLine($"{Settings.InfoColor}Lambda ARN:{Settings.ResetColor} {record.lambdaArn}");
var entries = await DecodeBase64GzipDataAsync(record.rawData);
Console.WriteLine($"{Settings.InfoColor}Entries:{Settings.ResetColor} {JObject.Parse(entries).ToString(Formatting.Indented)}");
}
} catch (Exception e) {
LogError($"unable to read records for {s3Object.Key}", e);
}
Console.WriteLine();
}
request.ContinuationToken = response.NextContinuationToken;
} while(request.ContinuationToken != null);
// local functions
async Task <IEnumerable <KinesisFailedLogRecord> > GetS3ObjectContentsAsync(string key)
{
try {
var response = await settings.S3Client.GetObjectAsync(new GetObjectRequest {
BucketName = settings.LoggingBucketName,
Key = key
});
using (var decompressionStream = new GZipStream(response.ResponseStream, CompressionMode.Decompress))
using (var destinationStream = new MemoryStream()) {
await decompressionStream.CopyToAsync(destinationStream);
return(Encoding.UTF8.GetString(destinationStream.ToArray())
.Split(SEPARATORS, StringSplitOptions.RemoveEmptyEntries)
.Select(json => JsonConvert.DeserializeObject <KinesisFailedLogRecord>(json.Trim())));
}
} catch (AmazonS3Exception) {
return(null);
}
}
async Task <string> DecodeBase64GzipDataAsync(string data)
{
using (var sourceStream = new MemoryStream(Convert.FromBase64String(data)))
using (var decompressionStream = new GZipStream(sourceStream, CompressionMode.Decompress))
using (var destinationStram = new MemoryStream()) {
await decompressionStream.CopyToAsync(destinationStram);
return(Encoding.UTF8.GetString(destinationStram.ToArray()));
}
}
}
public async Task <IEnumerable <StorageProviderItem> > GetChildrenByParentItem(StorageProviderItem parent)
{
using (var api = AmazonS3Helper.GetApi(_account))
{
if (parent.Id == "/")
{
var response = await api.ListBucketsAsync();
var items = response.Buckets.Select(_ => new StorageProviderItem
{
Id = _.BucketName,
Name = _.BucketName,
Type = StorageProviderItemType.Folder,
ParentReferenceId = parent.Id
});
return(items.ToArray());
}
else
{
string bucket;
string prefix;
GetBucketAndKey(parent.Id, out bucket, out prefix);
var request = new ListObjectsV2Request
{
BucketName = bucket,
Prefix = prefix,
Delimiter = "/",
};
var items = new List <StorageProviderItem>();
ListObjectsV2Response response;
do
{
response = await api.ListObjectsV2Async(request);
items.AddRange(response.CommonPrefixes.Select(folderName => new StorageProviderItem
{
Id = bucket + "/" + folderName,
Name = folderName.RemovePrefix(prefix).RemoveTrailingSlash(),
Type = StorageProviderItemType.Folder,
ParentReferenceId = parent.Id,
}));
foreach (var o in response.S3Objects)
{
var normalized = o.Key.RemovePrefix(prefix);
if (string.IsNullOrEmpty(normalized) || normalized.EndsWith("/")) // Is Parent Folder (Dummy Item)? => ignore
{
continue;
}
items.Add(new StorageProviderItem
{
Id = bucket + "/" + o.Key,
Name = normalized,
Type = StorageProviderItemType.File,
ParentReferenceId = parent.Id,
LastModifiedDateTime = o.LastModified
});
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
return(items.ToArray());
}
}
}
public ICustomActivityResult Execute()
{
string Message = string.Empty;
var dataTable = new DataTable("List Files", "AWSS3");
try
{
switch (keyRegion)
{
case "1":
regionEndpoint = RegionEndpoint.USEast2;
break;
case "2":
regionEndpoint = RegionEndpoint.USEast1;
break;
case "3":
regionEndpoint = RegionEndpoint.USWest1;
break;
case "4":
regionEndpoint = RegionEndpoint.USWest2;
break;
case "5":
regionEndpoint = RegionEndpoint.APEast1;
break;
case "6":
regionEndpoint = RegionEndpoint.APSouth1;
break;
case "7":
regionEndpoint = RegionEndpoint.APNortheast2;
break;
case "8":
regionEndpoint = RegionEndpoint.APSoutheast1;
break;
case "9":
regionEndpoint = RegionEndpoint.APSoutheast2;
break;
case "10":
regionEndpoint = RegionEndpoint.APNortheast2;
break;
case "11":
regionEndpoint = RegionEndpoint.CACentral1;
break;
case "12":
regionEndpoint = RegionEndpoint.CNNorth1;
break;
case "13":
regionEndpoint = RegionEndpoint.CNNorthWest1;
break;
case "14":
regionEndpoint = RegionEndpoint.EUCentral1;
break;
case "15":
regionEndpoint = RegionEndpoint.EUWest1;
break;
case "16":
regionEndpoint = RegionEndpoint.EUWest2;
break;
case "17":
regionEndpoint = RegionEndpoint.EUWest3;
break;
case "18":
regionEndpoint = RegionEndpoint.EUNorth1;
break;
case "19":
regionEndpoint = RegionEndpoint.MESouth1;
break;
case "20":
regionEndpoint = RegionEndpoint.SAEast1;
break;
case "21":
regionEndpoint = RegionEndpoint.USGovCloudEast1;
break;
case "22":
regionEndpoint = RegionEndpoint.USGovCloudWest1;
break;
}
client = new AmazonS3Client(AccessKey, SecretKey, regionEndpoint);
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = bucketName,
MaxKeys = 100
};
ListObjectsV2Response response;
do
{
response = client.ListObjectsV2(request);
dataTable.Columns.Add("Id");
dataTable.Columns.Add("FileName");
dataTable.Columns.Add("BucketName");
dataTable.Columns.Add("Size");
dataTable.Columns.Add("ETag");
dataTable.Columns.Add("LastModified");
dataTable.Columns.Add("Owner");
int id = 0;
foreach (var item in response.S3Objects)
{
id++;
dataTable.Rows.Add(id, item.Key, item.BucketName, item.Size, item.ETag, item.LastModified, item.Owner);
}
// Process the response.
request.ContinuationToken = response.NextContinuationToken;
}while (response.IsTruncated);
}
catch (AmazonS3Exception amazonS3Exception)
{
Message = "S3 error occurred. Exception: " + amazonS3Exception.ToString();
return(this.GenerateActivityResult(Message));
}
catch (Exception e)
{
Message = "Exception: " + e.ToString();
Console.ReadKey();
}
return(this.GenerateActivityResult(dataTable));
}
public WfResult Run(CancellationToken token)
{
WfResult result = WfResult.Unknown;
//_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING]));
int runId = 0;
Int32.TryParse(_attributes[ETL_RUNID], out runId);
int batchId = 0;
Int32.TryParse(_attributes[ETL_BATCHID], out batchId);
int stepId = 0;
Int32.TryParse(_attributes[ETL_STEPID], out stepId);
bool setCounterInd = !String.IsNullOrEmpty(_attributes[COUNTER_NAME]);
int count = 0;
Int32.TryParse(_attributes[COUNT], out count);
if (count <= 0)
{
count = MAX_COUNT;
}
ControllerCounter counter = new ControllerCounter(_attributes[CONNECTION_STRING], _logger)
{
BatchId = batchId,
StepId = stepId,
RunId = runId
};
try
{
//if(!Amazon.Util.ProfileManager.IsProfileKnown(_attributes[PROFILE_NAME]))
//{
// Amazon.Util.ProfileManager.RegisterProfile(_attributes[PROFILE_NAME], _attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]);
//}
//var credentials = new StoredProfileAWSCredentials(_attributes[PROFILE_NAME]);
RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(_attributes[REGION_NAME]);
AWSCredentials credentials = null;
if (!String.IsNullOrEmpty(_attributes[ACCOUNT_NAME]))
{
credentials = new BasicAWSCredentials(_attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]);
}
using (var client = new AmazonS3Client(credentials, endpoint))
{
//ListBucketsRequest bucketRequest = new ListBucketsRequest();
//ListBucketsResponse backetResponse;
//backetResponse = client.ListBuckets(bucketRequest);
//var buckets = backetResponse.Buckets;
//GetBucketLocationResponse bucketLocationResponse = client.GetBucketLocation(_attributes[CONTAINER_NAME]);
ListObjectsV2Request objectRequest = new ListObjectsV2Request
{
BucketName = _attributes[CONTAINER_NAME],
MaxKeys = 100,
Prefix = String.IsNullOrEmpty(_attributes[INPUT_PREFIX]) ? null : _attributes[INPUT_PREFIX]
};
List <S3Object> objectlist = new List <S3Object>();
ListObjectsV2Response objectResponse;
do
{
objectResponse = client.ListObjectsV2(objectRequest);
objectlist.AddRange(objectResponse.S3Objects);
objectRequest.ContinuationToken = objectResponse.NextContinuationToken;
} while (objectResponse.IsTruncated == true);
IEnumerable <S3Object> list = objectlist;
if (_attributes[SORT_ORDER].Equals(sortList[0], StringComparison.InvariantCultureIgnoreCase))
{
list = list.OfType <S3Object>().OrderBy(b => b.Key);
}
if (_attributes[SORT_ORDER].Equals(sortList[1], StringComparison.InvariantCultureIgnoreCase))
{
list = list.OfType <S3Object>().OrderByDescending(b => b.Key);
}
if (count > 0)
{
list = list.OfType <S3Object>().Take(count);
}
Dictionary <string, string> files = new Dictionary <string, string>();
Directory.CreateDirectory(_attributes[OUTPUT_FOLDER]);
int i = 0;
foreach (var blobItem in list.OfType <S3Object>())
{
token.ThrowIfCancellationRequested();
string outputFile = Path.Combine(_attributes[OUTPUT_FOLDER], Path.GetFileName(blobItem.Key));
if (File.Exists(outputFile))
{
continue;
}
GetObjectRequest blobRequest = new GetObjectRequest
{
BucketName = _attributes[CONTAINER_NAME],
Key = blobItem.Key
};
using (FileStream outputFileStream = File.Create(outputFile))
using (GetObjectResponse blobResponse = client.GetObject(blobRequest))
using (Stream responseStream = blobResponse.ResponseStream)
//using (StreamReader reader = new StreamReader(responseStream))
{
responseStream.CopyTo(outputFileStream);
//responseBody = reader.ReadToEnd();
}
_logger.Debug("downloaded: {File}", outputFile);
if (setCounterInd)
{
files.Add(String.Format("{0}_{1}", _attributes[COUNTER_NAME], i++), outputFile);
}
}
if (setCounterInd)
{
counter.SetCounters(files);
}
}
}
catch (Exception ex)
{
throw ex;
}
result = WfResult.Succeeded;
return(result);
}
private IEnumerable <string> GetParts()
{
var folder = string.Format("{0}/{1}/raw", Settings.Current.Building.Vendor,
Settings.Current.Building.Id);
var fileName = string.Empty;
foreach (var qd in Settings.Current.Building.SourceQueryDefinitions)
{
var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database,
Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchemaName);
if (qd.Persons == null)
{
continue;
}
if (string.IsNullOrEmpty(sql))
{
continue;
}
fileName = qd.FileName;
break;
}
if (string.IsNullOrEmpty(fileName))
{
fileName = Settings.Current.Building.SourceQueryDefinitions[0].FileName;
}
using (var client = new AmazonS3Client(Settings.Current.S3AwsAccessKeyId, Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
{
var request = new ListObjectsV2Request
{
BucketName = Settings.Current.Bucket,
Prefix = string.Format("{0}/{1}/{2}/{3}", folder, chunkId, fileName, fileName)
};
ListObjectsV2Response response;
do
{
response = client.ListObjectsV2(request);
var partIndexes = new HashSet <string>();
var fn = fileName;
foreach (var entry in response.S3Objects)
{
var end = entry.Key.LastIndexOf(fn, StringComparison.InvariantCultureIgnoreCase) + fn.Length;
var key = entry.Key.Replace(entry.Key.Substring(0, end), "");
key = key.Substring(0, key.IndexOf('_'));
partIndexes.Add(key);
}
foreach (var partIndex in partIndexes)
{
yield return(partIndex);
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
}
}
public async Task BulkMoveItems()
{
// ARRANGE
string bucket = sourceBucket;
//string bucket = "mhaken-9fc61dff-1893-42cc-a82f-8024960b158e";
Stopwatch sw = new Stopwatch();
int count = 10000;
try
{
sw.Start();
await CreateAndFillBucket(count);
sw.Stop();
Debug.Print($"Finished creating and filling bucket {bucket} in {sw.Elapsed}.");
sw.Reset();
ListObjectsV2Request list = new ListObjectsV2Request()
{
BucketName = bucket
};
sw.Start();
IEnumerable <S3Object> objects = (await client.ListAllObjectsAsync(list)).SelectMany(x => x.S3Objects);
// ACT
BulkMoveRequest request = new BulkMoveRequest(objects.Select(x => new CopyObjectRequest()
{
SourceBucket = x.BucketName,
DestinationBucket = x.BucketName,
SourceKey = x.Key,
DestinationKey = "moved_" + x.Key,
StorageClass = S3StorageClass.OneZoneInfrequentAccess,
}))
{
PreferMultipart = true
};
BulkCopyResponse response = await client.BulkMoveAsync(request);
sw.Stop();
File.WriteAllText("results.txt", $"Successfully moved {count} items in {sw.Elapsed}.");
// ASSERT
Assert.Equal(objects.Count(), response.SuccessfulOperations.Count);
}
catch (Exception e)
{
}
finally
{
await client.EmptyBucketAsync(bucket);
await client.DeleteBucketAsync(bucket);
}
}
public void Fill(AmazonS3Client client, string bucket, string prefix)
{
//var prefix = $"Lookup/Truven/{fileName}.txt";
var spliter = new StringSplitter(5);
var result = new List <string>();
var request = new ListObjectsV2Request
{
BucketName = bucket,
Prefix = prefix
};
ListObjectsV2Response response;
do
{
var responseTask = client.ListObjectsV2Async(request);
responseTask.Wait();
response = responseTask.Result;
foreach (var entry in response.S3Objects)
{
if (entry.Size > 20)
{
result.Add(entry.Key);
}
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
var getObjectRequest = new GetObjectRequest
{
BucketName = bucket,
Key = result[0]
};
var getObject = client.GetObjectAsync(getObjectRequest);
getObject.Wait();
using (var responseStream = getObject.Result.ResponseStream)
using (var bufferedStream = new BufferedStream(responseStream))
using (var gzipStream = new GZipStream(bufferedStream, CompressionMode.Decompress))
using (var reader = new StreamReader(gzipStream, Encoding.Default))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (!string.IsNullOrEmpty(line))
{
spliter.SafeSplit(line, '\t');
var sourceCode = GetStringValue(spliter.Results[0]);
if (!_lookup.ContainsKey(sourceCode))
{
_lookup.Add(sourceCode, new Dictionary <int, LookupValue>());
}
//if (!_lookupIgnoreCase.ContainsKey(sourceCode))
// _lookupIgnoreCase.Add(sourceCode, new HashSet<string>());
//_lookupIgnoreCase[sourceCode].Add(sourceCode);
int conceptId = -1;
if (int.TryParse(spliter.Results[1], out var cptId))
{
conceptId = cptId;
}
if (!_lookup[sourceCode].ContainsKey(conceptId))
{
if (!DateTime.TryParse(spliter.Results[3], out var validStartDate))
{
validStartDate = DateTime.MinValue;
}
if (!DateTime.TryParse(spliter.Results[4], out var validEndDate))
{
validEndDate = DateTime.MaxValue;
}
var lv = new LookupValue
{
ConceptId = conceptId,
SourceCode = sourceCode,
Domain = GetStringValue(spliter.Results[2]),
ValidStartDate = validStartDate,
ValidEndDate = validEndDate,
//Ingredients = new HashSet<int>()
};
if (spliter.Results.Length > 5)
{
//lv.SourceVocabularyId = GetStringValue(spliter.Results[5]);
lv.SourceConceptId = IsNullOrEmpty(spliter.Results[6])
? 0
: int.Parse(spliter.Results[6]);
//if (!DateTime.TryParse(spliter.Results[7], out var validStartDate2))
// validStartDate2 = DateTime.MinValue;
//if (!DateTime.TryParse(spliter.Results[8], out var validEndDate2))
// validEndDate2 = DateTime.MaxValue;
//lv.SourceValidStartDate = validStartDate2;
//lv.SourceValidEndDate = validEndDate2;
}
_lookup[sourceCode].Add(conceptId, lv);
}
if (spliter.Results.Length > 5)
{
if (!IsNullOrEmpty(spliter.Results[9]) &&
int.TryParse(spliter.Results[9], out var ingredient))
{
if (_lookup[sourceCode][conceptId].Ingredients == null)
{
_lookup[sourceCode][conceptId].Ingredients = new HashSet <int>();
}
_lookup[sourceCode][conceptId].Ingredients.Add(ingredient);
}
}
}
}
foreach (var v1 in _lookup.Values)
{
foreach (var v2 in v1.Values)
{
v2.Ingredients?.TrimExcess();
}
}
GC.Collect();
}
}
public async Task Lock(CancellationToken cancellationToken)
{
var nowTask = GetUtcTime(cancellationToken);
var tasks = new List <Task>();
var lockLifetime = TimeSpan.FromMinutes(3);
var listRequest = new ListObjectsV2Request
{
BucketName = _context.BucketName,
Prefix = S3Constants.LockPrefix,
};
while (true)
{
var listResponse = await _context.S3.Invoke((s3, token) => s3.ListObjectsV2Async(listRequest, token), cancellationToken);
var threshold = (await nowTask)-lockLifetime;
var locked = false;
foreach (var s3Object in listResponse.S3Objects)
{
var key = s3Object.Key;
var lockTime = s3Object.LastModified.ToUniversalTime();
if (lockTime < threshold)
{
var deleteObjectRequest = new DeleteObjectRequest
{
BucketName = _context.BucketName,
Key = key,
};
var deleteTask = _context.S3.Invoke((s3, token) => s3.DeleteObjectAsync(deleteObjectRequest, token), cancellationToken);
tasks.Add(deleteTask);
}
else if (key.EndsWith(LockExtension, StringComparison.OrdinalIgnoreCase))
{
var keyData = key.Substring(S3Constants.LockPrefix.Length, key.Length - LockExtension.Length - S3Constants.LockPrefix.Length);
if (keyData == _lockName)
{
locked = true;
break;
}
var message = _isLocked
? FormattableString.Invariant($"The lock \"{_lockName}\" was overriden by \"{keyData}\" (time: {lockTime:o}, threshold: {threshold:o}).")
: FormattableString.Invariant($"The lock \"{_lockName}\" is prevented by \"{keyData}\" (time: {lockTime:o}, threshold: {threshold:o}).");
_isLocked = false;
throw new OperationCanceledException(message);
}
listRequest.StartAfter = key;
}
if (locked || !listResponse.IsTruncated)
{
break;
}
}
var request = new PutObjectRequest
{
BucketName = _context.BucketName,
Key = S3Constants.LockPrefix + _lockName + LockExtension,
};
var putTask = _context.S3.Invoke((s3, token) => s3.PutObjectAsync(request, token), cancellationToken);
tasks.Add(putTask);
await Task.WhenAll(tasks);
_isLocked = true;
}
public static async Task <IActionResult> ListingObjectsAsync(HttpRequest req, TraceWriter log)
{
// Reference: https://docs.aws.amazon.com/AmazonS3/latest/dev/ListingObjectKeysUsingNetSDK.html
var bucketName = Environment.GetEnvironmentVariable("S3BucketName", EnvironmentVariableTarget.Process);
var bucketRegion = RegionEndpoint.GetBySystemName(Environment.GetEnvironmentVariable("S3BucketRegion", EnvironmentVariableTarget.Process));
if (awsSecretKey == null)
{
log.Info($"Fetching AWS secret key for the first time from KeyVault...");
var azureServiceTokenProvider = new AzureServiceTokenProvider();
var keyVaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(azureServiceTokenProvider.KeyVaultTokenCallback));
var secretName = Environment.GetEnvironmentVariable("AmazonS3SecretAccessKeySecretName", EnvironmentVariableTarget.Process);
var azureKeyVaultUrl = Environment.GetEnvironmentVariable("AzureKeyVaultUrl", EnvironmentVariableTarget.Process);
var secret = await keyVaultClient.GetSecretAsync($"{azureKeyVaultUrl}secrets/{secretName}").ConfigureAwait(false);
awsSecretKey = secret.Value;
log.Info("[Setting]: Successfully fetched AWS secret key from KeyVault.");
}
var credentials = new Amazon.Runtime.BasicAWSCredentials(Environment.GetEnvironmentVariable("AwsAccessKey", EnvironmentVariableTarget.Process), awsSecretKey);
string s3BucketLastProcessedDateTimeUtcAsString = req.Query["s3BucketLastProcessedDateTimeUtc"]; // GET
string requestBody = new StreamReader(req.Body).ReadToEnd(); // POST
dynamic data = JsonConvert.DeserializeObject(requestBody);
s3BucketLastProcessedDateTimeUtcAsString = s3BucketLastProcessedDateTimeUtcAsString ?? data?.s3BucketLastProcessedDateTimeUtc;
if (string.IsNullOrWhiteSpace(s3BucketLastProcessedDateTimeUtcAsString))
{
string errorMessage = "A 's3BucketLastProcessedDateTimeUtc' querystring parameter or a request body containing a JSON object with a 's3BucketLastProcessedDateTimeUtc' property was expected but not found.";
log.Info(errorMessage);
return(new BadRequestObjectResult(errorMessage));
}
var s3BucketLastProcessedDateTimeUtc = DateTime.Parse(s3BucketLastProcessedDateTimeUtcAsString);
log.Info($"Bucket Name: {bucketName}.");
log.Info($"Bucket Region: {bucketRegion}.");
log.Info($"S3 Bucket Last Processed DateTimeUtc: {s3BucketLastProcessedDateTimeUtcAsString}.");
List <S3Object> filteredObjects = new List <S3Object>();
int totalUnfilteredCount = 0;
int currentUnfilteredCount = 0;
DateTime newLastProcessedDateTimeUtc = DateTime.UtcNow;
IAmazonS3 client = new AmazonS3Client(credentials, bucketRegion);
try
{
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = bucketName
};
ListObjectsV2Response response;
do
{
response = await client.ListObjectsV2Async(request);
currentUnfilteredCount = response.S3Objects.Count;
totalUnfilteredCount += currentUnfilteredCount;
log.Info($"Results Count (pre-filtering): {currentUnfilteredCount}.");
var currentFilteredObjects = response.S3Objects.FindAll((s3Object) =>
{
// Return objects updated after the last process date and that are not folder records (end with _$folder$ and have 0 size).
return(DateTime.Compare(s3Object.LastModified.ToUniversalTime(), s3BucketLastProcessedDateTimeUtc) > 0
& !(s3Object.Key.EndsWith("_$folder$", StringComparison.InvariantCulture) && s3Object.Size == 0));
});
log.Info($"Results Count (post-filtering): {currentFilteredObjects.Count}.");
filteredObjects.AddRange(currentFilteredObjects);
log.Info($"Next Continuation Token: {response.NextContinuationToken}.");
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
log.Info($"Results Count (total-unfiltered): {totalUnfilteredCount}.");
log.Info($"Results Count (total-filtered): {filteredObjects.Count}.");
dynamic payload = new System.Dynamic.ExpandoObject();
payload.s3Objects = filteredObjects;
payload.newLastProcessedDateTimeUtc = newLastProcessedDateTimeUtc.ToString();
return(new OkObjectResult(JsonConvert.SerializeObject(payload, Formatting.Indented)));
}
catch (AmazonS3Exception amazonS3Exception)
{
log.Info($"AmazonS3Exception [ListingObjectsAsync]: {amazonS3Exception.ToString()}.");
return(new BadRequestObjectResult("Operation failed (AmazonS3Exception). Check function's log for details."));
}
catch (Exception exception)
{
log.Info($"Exception [ListingObjectsAsync]: {exception.ToString()}.");
return(new BadRequestObjectResult("Operation failed. Check function's log for details."));
}
}
public async Task <ModuleLocation> ResolveInfoToLocationAsync(
ModuleInfo moduleInfo,
string bucketName,
ModuleManifestDependencyType dependencyType,
bool allowImport,
bool showError
)
{
if (bucketName == null)
{
throw new ArgumentNullException(nameof(bucketName));
}
LogInfoVerbose($"... resolving module {moduleInfo}");
StartLogPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}");
var cached = false;
try {
// check if module can be found in the deployment bucket
var result = await FindNewestModuleVersionInBucketAsync(Settings.DeploymentBucketName);
// check if the origin bucket needs to be checked
if (
allowImport &&
(Settings.DeploymentBucketName != bucketName) &&
(
// no version has been found
(result.Version == null)
// no module version constraint was given; the ultimate floating version
|| (moduleInfo.Version == null)
// the module version constraint is for a pre-release; we always prefer the origin version then
|| moduleInfo.Version.IsPreRelease()
// the module version constraint is floating; we need to check if origin has a newer version
|| !moduleInfo.Version.Minor.HasValue ||
!moduleInfo.Version.Patch.HasValue
)
)
{
var originResult = await FindNewestModuleVersionInBucketAsync(bucketName);
// check if module found at origin should be kept instead
if (
(originResult.Version != null) &&
(
(result.Version == null) ||
(moduleInfo.Version?.IsPreRelease() ?? false) ||
originResult.Version.IsGreaterThanVersion(result.Version)
)
)
{
result = originResult;
}
}
// check if a module was found
if (result.Version == null)
{
// could not find a matching version
var versionConstraint = (moduleInfo.Version != null)
? $"v{moduleInfo.Version} or later"
: "any released version";
if (showError)
{
if (allowImport)
{
LogError($"could not find module '{moduleInfo}' ({versionConstraint})");
}
else
{
LogError($"missing module dependency must be imported explicitly '{moduleInfo}' ({versionConstraint})");
}
}
return(null);
}
LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}");
return(MakeModuleLocation(result.Origin, result.Manifest));
} finally {
StopLogPerformance(cached);
}
async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionInBucketAsync(string bucketName)
{
StartLogPerformance($"FindNewestModuleVersionInBucketAsync() for s3://{bucketName}");
try {
// enumerate versions in bucket
var found = await FindModuleVersionsInBucketAsync(bucketName);
if (!found.Any())
{
return(Origin : bucketName, Version : null, Manifest : null);
}
// NOTE (2019-08-12, bjorg): if the module is nested or root, we filter the list of found versions to
// only contain versions that meet the module version constraint; for shared modules, we want to
// keep the latest version that is compatible with the tool and is equal-or-greater than the
// module version constraint.
if (
(
(dependencyType == ModuleManifestDependencyType.Root) ||
(dependencyType == ModuleManifestDependencyType.Nested)
) && (moduleInfo.Version != null)
)
{
found = found.Where(version => {
if (!version.MatchesConstraint(moduleInfo.Version))
{
LogInfoVerbose($"... rejected v{version}: does not match version constraint {moduleInfo.Version}");
return(false);
}
return(true);
}).ToList();
}
// attempt to identify the newest module version compatible with the tool
ModuleManifest manifest = null;
var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidateVersion => {
var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidateVersion, moduleInfo.Origin);
// check if the module version is allowed by the build policy
if (!(Settings.BuildPolicy?.Modules?.Allow?.Contains(candidateModuleInfo.ToString()) ?? true))
{
LogInfoVerbose($"... rejected v{candidateVersion}: not allowed by build policy");
return(false);
}
// load module manifest
var(candidateManifest, candidateManifestErrorReason) = LoadManifestFromLocationAsync(new ModuleLocation(bucketName, candidateModuleInfo, "<MISSING>")).GetAwaiter().GetResult();
if (candidateManifest == null)
{
LogInfoVerbose($"... rejected v{candidateVersion}: {candidateManifestErrorReason}");
return(false);
}
// check if module is compatible with this tool
if (!VersionInfoCompatibility.IsModuleCoreVersionCompatibleWithToolVersion(candidateManifest.CoreServicesVersion, Settings.ToolVersion))
{
LogInfoVerbose($"... rejected v{candidateVersion}: not compatible with tool version {Settings.ToolVersion}");
return(false);
}
// keep this manifest
manifest = candidateManifest;
return(true);
});
return(Origin : bucketName, Version : match, Manifest : manifest);
} finally {
StopLogPerformance();
}
}
async Task <IEnumerable <VersionInfo> > FindModuleVersionsInBucketAsync(string bucketName)
{
StartLogPerformance($"FindModuleVersionsInBucketAsync() for s3://{bucketName}");
var cached = false;
try {
var moduleOrigin = moduleInfo.Origin ?? Settings.DeploymentBucketName;
List <VersionInfo> versions = null;
string region = null;
// check if a cached version exists
string cachedManifestVersionsFilePath = null;
if (!Settings.ForceRefresh)
{
var cachedManifestFolder = GetCachedManifestDirectory(bucketName, moduleOrigin, moduleInfo.Namespace, moduleInfo.Name);
if (cachedManifestFolder != null)
{
cachedManifestVersionsFilePath = Path.Combine(cachedManifestFolder, "versions.json");
if (
File.Exists(cachedManifestVersionsFilePath) &&
(File.GetLastWriteTimeUtc(cachedManifestVersionsFilePath).Add(Settings.CachedManifestListingExpiration) > DateTime.UtcNow)
)
{
cached = true;
var cachedManifestVersions = JsonSerializer.Deserialize <ModuleManifestVersions>(File.ReadAllText(cachedManifestVersionsFilePath), Settings.JsonSerializerOptions);
region = cachedManifestVersions.Region;
versions = cachedManifestVersions.Versions;
}
}
}
// check if data needs to be fetched from S3 bucket
if (versions == null)
{
// get bucket region specific S3 client
var s3Client = await GetS3ClientByBucketNameAsync(bucketName);
if (s3Client == null)
{
// nothing to do; GetS3ClientByBucketName already emitted an error
return(new List <VersionInfo>());
}
// enumerate versions in bucket
versions = new List <VersionInfo>();
region = s3Client.Config.RegionEndpoint.SystemName;
var request = new ListObjectsV2Request {
BucketName = bucketName,
Prefix = $"{moduleOrigin}/{moduleInfo.Namespace}/{moduleInfo.Name}/",
Delimiter = "/",
MaxKeys = 100,
RequestPayer = RequestPayer.Requester
};
do
{
try {
var response = await s3Client.ListObjectsV2Async(request);
versions.AddRange(response.S3Objects
.Select(s3Object => s3Object.Key.Substring(request.Prefix.Length))
.Select(found => VersionInfo.Parse(found))
);
request.ContinuationToken = response.NextContinuationToken;
} catch (AmazonS3Exception e) when(e.Message == "Access Denied")
{
// show message that access was denied for this location
LogInfoVerbose($"... access denied to {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]");
return(Enumerable.Empty <VersionInfo>());
}
} while(request.ContinuationToken != null);
// cache module versions listing
if (cachedManifestVersionsFilePath != null)
{
try {
File.WriteAllText(cachedManifestVersionsFilePath, JsonSerializer.Serialize(new ModuleManifestVersions {
Region = region,
Versions = versions
}, Settings.JsonSerializerOptions));
} catch {
// nothing to do
}
}
}
// filter list down to matching versions
versions = versions.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true)).ToList();
LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{region}]");
return(versions);
} finally {
StopLogPerformance(cached);
}
}
ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest)
=> new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum);
}
public async Task <ListObjectsV2Response> ListObjectsV2Async(ListObjectsV2Request request) => await ExecuteS3Request(() => _s3Client.ListObjectsV2Async(request), request.BucketName);
/// <summary>
/// This routine will retireve all objects within the specified S3 Bucket
/// Any objects which have a modified date older than our target will be removed
/// </summary>
/// <param name="modifiedDate"></param>
/// <returns></returns>
private static async Task <List <S3Object> > ListObjects(DateTime modifiedDate)
{
try
{
List <S3Object> s3Objects = new List <S3Object>();
ListObjectsV2Request request = new ListObjectsV2Request()
{
BucketName = bucketName
};
ListObjectsV2Response response;
Console.WriteLine("Connecting to S3...");
do
{
response = await client.ListObjectsV2Async(request);
foreach (S3Object entry in response.S3Objects)
{
if (entry.LastModified.IsAfter(modifiedDate))
{
var listResponse = client.ListVersionsAsync(new ListVersionsRequest
{
BucketName = bucketName,
Prefix = entry.Key
});
S3ObjectVersion deleteMarkerVersion = listResponse.Result.Versions.FirstOrDefault(x => x.IsLatest);
if (deleteMarkerVersion != null)
{
await client.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = bucketName,
Key = entry.Key,
VersionId = deleteMarkerVersion.VersionId
});
}
s3Objects.Add(entry);
}
}
Console.WriteLine("Next Continuation Token: {0}", response.NextContinuationToken);
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
return(s3Objects);
}
catch (AmazonS3Exception e)
{
Console.WriteLine("Error encountered on server. Message:'{0}' when writing an object", e.Message);
return(null);
}
catch (Exception e)
{
Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message);
return(null);
}
}
async Task <ListObjectsV2Response> IS3Client.ListObjectsV2Async(ListObjectsV2Request request) => await _amazonS3.ListObjectsV2Async(request);
private async Task GetObjectList(string BucketName, string Prefix)
{
this.StartSpinnerFrame();
this.BucketName = BucketName;
this.FolderKey = Prefix;
this.NavigationPanel.Children.Clear();
this.NavigationPanel.Children.Add(this.GetHyperLinkLabel(BucketName, "", RefreshFolder));
if (this.FolderKey != "")
{
string[] FolderKeys = this.FolderKey.Split('/');
string FolderKeyOrigin = "";
foreach (string Key in FolderKeys)
{
FolderKeyOrigin += Key + "/";
if (Key != "")
{
this.NavigationPanel.Children.Add(new Label {
Content = " > "
});
int iChild = this.NavigationPanel.Children.Add(this.GetHyperLinkLabel(FolderKey, FolderKeyOrigin, RefreshFolder));
}
}
}
var request = new ListObjectsV2Request
{
BucketName = BucketName
,
Prefix = Prefix
,
Delimiter = "/"
};
ObjectList.Items.Clear();
ListObjectsV2Response response;
do
{
response = await this.s3ClientHandler.ListObjectsV2Async(request);
foreach (string commonPrefix in response.CommonPrefixes)
{
var s3Objects = new S3Objects
{
Image = ResourceHelper.GetBitmapFrameForResourceImage(S3Browser.Properties.Resources.blue_folder)
,
Type = "Folder"
,
BucketName = BucketName
,
Key = commonPrefix
,
Name = this.FolderKey.Length > 0 ? commonPrefix.Replace(this.FolderKey, "") : commonPrefix
};
ListViewItem item = new ListViewItem
{
Content = s3Objects
};
_ = this.ObjectList.Items.Add(item);
}
foreach (S3Object s3Object in response.S3Objects)
{
var s3Objects = new S3Objects
{
Image = ResourceHelper.GetBitmapFrameForResourceImage(S3Browser.Properties.Resources.document)
,
Type = "Object"
,
BucketName = s3Object.BucketName
,
Key = s3Object.Key
,
Name = this.FolderKey.Length > 0 ? s3Object.Key.Replace(this.FolderKey, "") : s3Object.Key
,
LastModified = s3Object.LastModified
,
Owner = s3Object.Owner
,
StorageClass = s3Object.StorageClass
};
s3Objects.SetSize(s3Object.Size);
_ = this.ObjectList.Items.Add(new ListViewItem {
Content = s3Objects
});
}
request.ContinuationToken = response.NextContinuationToken;
this.EndSpinnerFrame();
} while (response.IsTruncated);
}
public List <FileListModel> GetDirectoryList(string directory, string accessToken)
{
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USWest1, // MUST set this before setting ServiceURL and it should match the `MINIO_REGION` enviroment variable.
ForcePathStyle = true // MUST be true to work correctly with Minio server
};
string region = accessToken.Split("-")[2];
if (region == "USWest1")
{
config.RegionEndpoint = RegionEndpoint.USWest1;
}
else if (region == "USWest2")
{
config.RegionEndpoint = RegionEndpoint.USWest2;
}
else if (region == "APNortheast1")
{
config.RegionEndpoint = RegionEndpoint.APNortheast1;
}
else if (region == "APNortheast2")
{
config.RegionEndpoint = RegionEndpoint.APNortheast2;
}
else if (region == "APSouth1")
{
config.RegionEndpoint = RegionEndpoint.APSouth1;
}
else if (region == "APSoutheast1")
{
config.RegionEndpoint = RegionEndpoint.APSoutheast1;
}
else if (region == "APSoutheast2")
{
config.RegionEndpoint = RegionEndpoint.APSoutheast2;
}
else if (region == "CACentral1")
{
config.RegionEndpoint = RegionEndpoint.CACentral1;
}
else if (region == "CNNorth1")
{
config.RegionEndpoint = RegionEndpoint.CNNorth1;
}
else if (region == "CNNorthWest1")
{
config.RegionEndpoint = RegionEndpoint.CNNorthWest1;
}
else if (region == "EUCentral1")
{
config.RegionEndpoint = RegionEndpoint.EUCentral1;
}
else if (region == "EUWest2")
{
config.RegionEndpoint = RegionEndpoint.EUWest2;
}
else if (region == "EUWest3")
{
config.RegionEndpoint = RegionEndpoint.EUWest3;
}
else if (region == "SAEast1")
{
config.RegionEndpoint = RegionEndpoint.SAEast1;
}
else if (region == "USEast1")
{
config.RegionEndpoint = RegionEndpoint.USEast1;
}
else if (region == "USEast2")
{
config.RegionEndpoint = RegionEndpoint.USEast2;
}
else if (region == "EUWest1")
{
config.RegionEndpoint = RegionEndpoint.EUWest1;
}
else if (region == "USGovCloudWest1")
{
config.RegionEndpoint = RegionEndpoint.USGovCloudWest1;
}
//var amazonS3Client = new AmazonS3Client("AKIAIZ7ZNECIUB4MEKRQ", "g2OZECVv8kRsbixfiP52xnKhHo/FSny5hm0qmb4q", config);
var amazonS3Client = new AmazonS3Client(accessToken.Split("-")[0], accessToken.Split("-")[1], config);
List <FileListModel> list = new List <FileListModel>();
if (String.IsNullOrEmpty(directory))
{
var listBucketResponse = (amazonS3Client.ListBucketsAsync()).Result;
foreach (var item in listBucketResponse.Buckets)
{
list.Add(new FileListModel()
{
CloudId = item.BucketName, Directory = item.BucketName
});
}
}
else
{
string bucket = Regex.Split(directory, "/")[0];
string innerDirectory = "";
string[] directoryArray = directory.Split("/");
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = bucket,
Delimiter = @"/",
};
if (directoryArray.Length > 1)
{
for (int i = 1; i < directoryArray.Length - 1; i++)
{
innerDirectory = directoryArray[i] + "/";
}
request.Prefix = innerDirectory;
}
else
{
}
var bucketDirectory = (amazonS3Client.ListObjectsV2Async(request)).Result;
foreach (var item in bucketDirectory.CommonPrefixes)
{
list.Add(new FileListModel()
{
CloudId = bucket + "/" + item, Directory = bucket + "/" + item
});
}
}
return(list);
}
/// <inheritdoc/>
public Task <ListObjectsV2Response> ListObjectsV2Async(ListObjectsV2Request request, CancellationToken cancellationToken = default(CancellationToken))
{
throw new NotImplementedException();
}
public void ListObjectsV2Async(ListObjectsV2Request request, AmazonServiceCallback <ListObjectsV2Request, ListObjectsV2Response> callback, AsyncOptions options = null)
{
throw new System.NotImplementedException();
}
protected void btnUpload_Click(object sender, EventArgs e)
{
/*
* First upload image in bucket
* detect face while uploading image into bucket
* compare face in face collection
* get list of image from bucket
* store uploaded image in face collection
*
* compare face in face collection while uploading
*
*
* */
try
{
image.Visible = false;
lblMessage.Text = "";
string imagefile = string.Empty;
if (fuImage.HasFile)
{
string imagename = System.IO.Path.GetFullPath(fuImage.PostedFile.FileName);
//Label1.Text = imagename;
string ext = System.IO.Path.GetExtension(fuImage.FileName);
// Label2.Text = ext;
// imagefile = Server.MapPath("~/Images/" + imagename);
imagefile = imagename;
if (ext == ".jpg" | ext == ".png")
{
//fuImage.SaveAs(imagefile);
Stream st = fuImage.PostedFile.InputStream;
IAmazonRekognition rekoClient = new AmazonRekognitionClient(Amazon.RegionEndpoint.USEast1);
string name = Path.GetFileName(fuImage.PostedFile.FileName);
name = @"C:\Blazar\CompareImage\" + fuImage.FileName;
string myBucketName = "blazarstorage"; //your s3 bucket name goes here
string s3DirectoryName = "";
string s3FileName = @name;
string fileName = fuImage.FileName;
/*
* StoreFaceInCollection(@name, rekoClient, myBucketName);
*
* ImageRecognition imageRecog = new ImageRecognition();
* IdentifyFaces(name, fuImage.PostedFile.FileName.ToString(), myBucketName);
* */
//STEP----1
//validate the image is a face or not--step 1
//bool isFaceImage = DetectFaces(fileName);
//bool isFaceImage = DetectFaces(name);
bool isFaceImage = true;
if (isFaceImage == true)
{
/*
* we can compare image if those are inside buckets.
* For comparing images , we have to follow below steps
* store it in bucket
* add to it face collection
* compare images
*
*/
//
bool isUpload;
//upload image into bucket
isUpload = sendMyFileToS3(st, myBucketName, s3DirectoryName, s3FileName, fileName);
if (isUpload == true)
{
//store image in a face collection
StoreFaceInCollection(@name, rekoClient, myBucketName);
// AmazonUploader myUploader = new AmazonUploader();
// bool b = myUploader.IsFilExist("AKIAIPXX2OIGXNKENL6Q", "oQQn3l4ll5/J/OY2RoZG4EV4RZtv8EsD114MrRnR");
//validate the existance of the image in the bucket
//for that we have to get all the face from face collection
try
{
AmazonS3Client s3Client = new AmazonS3Client(Amazon.RegionEndpoint.USEast1);
ListObjectsV2Request reqGetObjFromBucket = new ListObjectsV2Request
{
BucketName = myBucketName,
MaxKeys = 1000
};
ListObjectsV2Response resGetObjFromBucket;
do
{
resGetObjFromBucket = s3Client.ListObjectsV2(reqGetObjFromBucket);
foreach (Amazon.S3.Model.S3Object entry in resGetObjFromBucket.S3Objects)
{
//if (DetectFaces(entry.Key))//validat the image content
//{
// if (s3FileName != entry.Key)//input image should not compare
if (fileName != entry.Key)
{
var response = rekoClient.CompareFaces(new CompareFacesRequest
{
SimilarityThreshold = 90,
SourceImage = new Amazon.Rekognition.Model.Image
{
S3Object = new Amazon.Rekognition.Model.S3Object
{
Bucket = myBucketName,
// Name=""
// Name = s3FileName
Name = fileName
}
},
TargetImage = new Amazon.Rekognition.Model.Image
{
S3Object = new Amazon.Rekognition.Model.S3Object
{
Bucket = myBucketName,
// Name=""
Name = entry.Key
}
}
});
if (response.FaceMatches.Count > 0)
{
image.Visible = true;
dupImage.Src = name;
existingImage.Src = "https://s3.amazonaws.com/blazarstorage/" + entry.Key;;
lblMessage.Text = "You are trying to upload the image " + s3FileName + " which is matching with " + entry.Key;
lblMessage.ForeColor = System.Drawing.Color.Green;
IAmazonS3 s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast1);
s3.DeleteObject(myBucketName, fileName);
return;
}
}
// }
}
} while (resGetObjFromBucket.IsTruncated == true);
}
catch (Exception ex)
{
IAmazonS3 s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast1);
s3.DeleteObject(myBucketName, fileName);
lblMessage.Text = ex.Message.ToString();
lblMessage.ForeColor = System.Drawing.Color.Red;
}
// List< Face > lstOfFaces= GetListOfFaceInFaceCollection();
//if (lstOfFaces.Count >= 1)
//{
// for(int i=0;i<lstOfFaces.Count;i++)
// {
// GetObjectRequest objReq = new GetObjectRequest();
// GetObjectResponse res = new GetObjectResponse();
// if (response.FaceMatches.Count > 0)
// {
// IAmazonS3 s3 = new AmazonS3Client();
// s3.DeleteObject(myBucketName, s3FileName);
// }
// }
//}
}
else
{
lblMessage.Text = "Please upload again!";
}
//else
//{
// //upload image into bucket--step 3
// isUpload = sendMyFileToS3(st, myBucketName, s3DirectoryName, s3FileName);
// if (isUpload == true)
// {
// //store image in a face collection
// StoreFaceInCollection(@name, rekoClient, myBucketName);
// lblMessage.Text = "successfully uploaded";
// Response.Write("successfully uploaded");
// }
// else
// lblMessage.Text = "error";
//}
}
else
{
lblMessage.Text = "Please upload a valid image!!!";
}
}
}
}
catch (Exception ex)
{
lblMessage.Text = ex.Message.ToString();
}
}
// Please set the following connection strings in app.config for this WebJob to run:
// AzureWebJobsDashboard and AzureWebJobsStorage
private static void Main()
{
#region Parse Configuration Settings
AwsSettings awsSettings = new AwsSettings();
string AWS_SETTINGS = CloudConfigurationManager.GetSetting("AWS_SETTINGS");
awsSettings = JsonConvert.DeserializeObject <AwsSettings>(AWS_SETTINGS);
azureStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureStorageConnectionString"));
azureStorageContainerName = CloudConfigurationManager.GetSetting("AzureStorageContainerName");
string runMode = CloudConfigurationManager.GetSetting("runMode");
if (runMode == "INTERACTIVE")
{
interactiveMode = true;
}
#endregion
Console.WriteLine("CopyS3toBlob: {0}.", awsSettings.Description);
azureClient = azureStorageAccount.CreateCloudBlobClient();
azureClient.GetContainerReference(azureStorageContainerName).CreateIfNotExists();
// For each account in AwsSettings
foreach (var account in awsSettings.Accounts)
{
ConsoleColor currentColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine(@"Processing AWS Account {0}", account);
Console.ForegroundColor = currentColor;
// For each bucket we need to inspect for this account
foreach (var bucket in account.Buckets)
{
Console.WriteLine(@"Processing Bucket {0}", bucket.Name);
count = 0;
files = new Dictionary <string, S3Object>();
Amazon.RegionEndpoint region = GetAwsRegionFromString(bucket.Region);
using (client = new AmazonS3Client(account.AccessKey, account.SecretAccessKey, region))
{
try
{
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = bucket.Name,
MaxKeys = 10,
Prefix = account.AccountId
};
ListObjectsV2Response response;
do
{
response = client.ListObjectsV2(request);
// Process response.
foreach (S3Object entry in response.S3Objects)
{
if (FileIsRequired(entry.Key, bucket.Patterns))
{
count++;
//Console.WriteLine("key = {0} size = {1}, Date={2}", entry.Key, entry.Size, entry.LastModified);
S3Object theFile = new S3Object();
theFile.BucketName = entry.BucketName;
theFile.ETag = entry.ETag;
theFile.LastModified = entry.LastModified;
theFile.Owner = entry.Owner;
theFile.Size = entry.Size;
theFile.StorageClass = entry.StorageClass;
files.Add(entry.Key, theFile);
}
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated == true);
}
catch (AmazonS3Exception amazonS3Exception)
{
if (amazonS3Exception.ErrorCode != null &&
(amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId")
||
amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
{
Console.WriteLine("Check the provided AWS Credentials.");
Console.WriteLine(
"To sign up for service, go to http://aws.amazon.com/s3");
}
else
{
Console.WriteLine(
"Error occurred. Message:'{0}' when listing objects",
amazonS3Exception.Message);
}
throw amazonS3Exception;
}
}
ConsoleColor savedColour = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Bucket{0} contains {1} files of interest:",
bucket.Name, count);
Console.ForegroundColor = savedColour;
BucketBlobPackage package = new BucketBlobPackage();
package.AccountId = account.AccountId;
package.AccessKey = account.AccessKey;
package.SecretAccessKey = account.SecretAccessKey;
package.BucketInfo = bucket;
package.FileInfo = files;
WriteToBlobStorage(package);
}
}
Console.WriteLine("CopyS2toBlob job has completed sucessfully");
if (interactiveMode)
{
Console.ReadKey();
}
}
public async Task <ListObjectsV2Response> ListObjectsV2Async(ListObjectsV2Request request) => await _s3Client.ListObjectsV2Async(request);
/// <summary>
///
/// <para>DeleteFolder:</para>
///
/// <para>Deletes a folder from File Service, caller thread will be blocked before it is done</para>
///
/// <para>Check <seealso cref="IBFileServiceInterface.DeleteFile"/> for detailed documentation</para>
///
/// </summary>
public bool DeleteFolder(
string _BucketName,
string _Folder,
Action <string> _ErrorMessageAction = null)
{
if (S3Client == null)
{
_ErrorMessageAction?.Invoke("BFileServiceAWS->DeleteFolder: S3Client is null.");
return(false);
}
var PrefixedObjects = new List <KeyVersion>();
try
{
ListObjectsV2Request ListRequest = new ListObjectsV2Request()
{
BucketName = _BucketName,
Prefix = _Folder
};
using (var ListObjectsTask = S3Client.ListObjectsV2Async(ListRequest))
{
ListObjectsTask.Wait();
foreach (var FileObject in ListObjectsTask.Result.S3Objects)
{
if (FileObject != null)
{
PrefixedObjects.Add(new KeyVersion()
{
Key = FileObject.Key
});
}
}
}
}
catch (Exception e)
{
PrefixedObjects = null;
_ErrorMessageAction?.Invoke("BFileServiceAWS->DeleteFolder-ListPrefixed: " + e.Message + ", Trace: " + e.StackTrace);
return(false);
}
DeleteObjectsRequest Request = new DeleteObjectsRequest
{
BucketName = _BucketName,
Objects = PrefixedObjects
};
try
{
using (var CreatedTask = S3Client.DeleteObjectsAsync(Request))
{
CreatedTask.Wait();
}
}
catch (Exception e)
{
_ErrorMessageAction?.Invoke("BFileServiceAWS->DeleteFolder: " + e.Message + ", Trace: " + e.StackTrace);
return(false);
}
return(true);
}
public async Task <ListObjectsV2Response> GetListAsync(ListObjectsV2Request request)
{
return(await _s3.ListObjectsV2Async(request));
}
public async Task <IActionResult> GetListingAsync(ListObjectsV2Request request)
{
var response = await _fileServices.GetListAsync(request);
return(Ok(response));
}
static async Task <List <KeyVersion> > ListingObjectsAsync()
{
List <KeyVersion> keyVersion = new List <KeyVersion>();
bool isContinue = false;
try
{
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = bucketName,
MaxKeys = 20,
Delimiter = "/",
Prefix = prefix // "private/events/" + eventCode +"/sessions/"
};
ListObjectsV2Response response;
do
{
response = await s3Client.ListObjectsV2Async(request);
List <KeyVersion> tempkeyVersion = new List <KeyVersion>();
int counter = 1;
// Process the response.
foreach (S3Object entry in response.S3Objects)
{
Console.WriteLine("id = {0} key = {1} size = {2}",
counter, entry.Key, entry.Size);
tempkeyVersion.Add(new KeyVersion()
{
Key = entry.Key, VersionId = counter.ToString()
});
counter++;
}
//Console.WriteLine("Next Continuation Token: {0}", response.NextContinuationToken);
request.ContinuationToken = response.NextContinuationToken;
//take comma seperated list ok item and put to List
//then ask for new list or delete
isContinue = false;
Console.WriteLine("1. Enter 1 for entering comma seperated list of objects to be deleted");
Console.WriteLine("2. Fetch new list of assets");
string input = Console.ReadLine();
try
{
int action = Convert.ToInt32(input);
switch (action)
{
case 1:
Console.WriteLine("Enter asset id corresponding to asset item");
input = Console.ReadLine();
var tagIds = new List <string>(input.Split(','));
foreach (var item in tagIds)
{
var query = tempkeyVersion.Where(x => x.VersionId == item).FirstOrDefault();
keyVersion.Add(new KeyVersion
{
Key = query.Key,
VersionId = "1"
});
}
break;
case 2:
isContinue = true;
break;
default:
throw new Exception();
break;
}
}
catch (Exception ex)
{
Console.WriteLine("No such command.");
}
} while (response.IsTruncated && isContinue);
}
catch (AmazonS3Exception amazonS3Exception)
{
Console.WriteLine("S3 error occurred. Exception: " + amazonS3Exception.ToString());
Console.ReadKey();
}
catch (Exception e)
{
Console.WriteLine("Exception: " + e.ToString());
Console.ReadKey();
}
return(keyVersion);
}
public async Task <int> DeleteFilesAsync(string searchPattern = null, CancellationToken cancellationToken = new CancellationToken())
{
var criteria = GetRequestCriteria(searchPattern);
int count = 0;
const int PAGE_SIZE = 100;
var listRequest = new ListObjectsV2Request {
BucketName = _bucket, Prefix = criteria.Prefix, MaxKeys = PAGE_SIZE
};
var deleteRequest = new DeleteObjectsRequest {
BucketName = _bucket
};
var errors = new List <DeleteError>();
ListObjectsV2Response listResponse;
do
{
listResponse = await _client.ListObjectsV2Async(listRequest, cancellationToken).AnyContext();
listRequest.ContinuationToken = listResponse.NextContinuationToken;
var keys = listResponse.S3Objects.MatchesPattern(criteria.Pattern).Select(o => new KeyVersion {
Key = o.Key
}).ToArray();
if (keys.Length == 0)
{
continue;
}
deleteRequest.Objects.AddRange(keys);
var deleteResponse = await _client.DeleteObjectsAsync(deleteRequest, cancellationToken).AnyContext();
if (deleteResponse.DeleteErrors.Count > 0)
{
// retry 1 time, continue on.
var deleteRetryRequest = new DeleteObjectsRequest {
BucketName = _bucket
};
deleteRetryRequest.Objects.AddRange(deleteResponse.DeleteErrors.Select(e => new KeyVersion {
Key = e.Key
}));
var deleteRetryResponse = await _client.DeleteObjectsAsync(deleteRetryRequest, cancellationToken).AnyContext();
if (deleteRetryResponse.DeleteErrors.Count > 0)
{
errors.AddRange(deleteRetryResponse.DeleteErrors);
}
}
count += deleteResponse.DeletedObjects.Count;
deleteRequest.Objects.Clear();
} while (listResponse.IsTruncated && !cancellationToken.IsCancellationRequested);
if (errors.Count > 0)
{
int more = errors.Count > 20 ? errors.Count - 20 : 0;
throw new Exception($"Unable to delete all S3 entries \"{String.Join(",", errors.Take(20).Select(e => e.Key))}\"{(more > 0 ? $" plus {more} more" : "")}.");
}
return(count);
}
private async Task<IEnumerable<SourceImage>> ListImagesAsync(
ServiceProvider provider)
{
var entries = await this.GetImageMapAsync(provider.GetService<BucketMonitorContext>());
var results = new List<SourceImage>();
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = this.BucketName
};
var console = new ConsoleString();
console.Update($"Loading Objects...");
int cacheAdditions = 0;
int count = 0;
ListObjectsV2Response response;
do
{
using (var scoped = provider.CreateScope())
{
var dbContext = scoped.ServiceProvider.GetService<BucketMonitorContext>();
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
var tracked = await this.GetTrackedBucket(dbContext);
response = await this.Client.ListObjectsV2Async(request);
var images = new List<SourceImage>();
var toAdd = new List<ImageEntry>();
var filtered = response.S3Objects
.Where(x => x.Key.Length <= MAX_PATH_LENGTH);
foreach (var obj in filtered)
{
var image = this.ProcessObject(
tracked: tracked,
obj: obj,
entries: entries,
toAdd: toAdd);
images.Add(image);
}
if (toAdd.Count() > 0)
{
dbContext.Image.AddRange(toAdd);
await dbContext.SaveChangesAsync();
cacheAdditions += toAdd.Count();
}
results.AddRange(images);
request.ContinuationToken = response.NextContinuationToken;
count++;
if (cacheAdditions > 0)
{
console.Update($"Loading Objects: {results.Count()}, CacheAdditions={cacheAdditions}");
}
else
{
console.Update($"Loading Objects: {results.Count()}");
}
}
}
while (response.IsTruncated);
console.Update($"Loaded {results.Count()} Objects");
this.Logger.LogDebug($"Queried {results.Count()} Objects");
return results;
}
public async Task <ModuleLocation> ResolveInfoToLocationAsync(ModuleInfo moduleInfo, ModuleManifestDependencyType dependencyType, bool allowImport, bool showError, bool allowCaching = false)
{
LogInfoVerbose($"... resolving module {moduleInfo}");
var stopwatch = Stopwatch.StartNew();
var cached = false;
try {
// check if a cached manifest matches
var cachedDirectory = Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo));
if (allowCaching && Settings.AllowCaching && Directory.Exists(cachedDirectory))
{
var foundCached = Directory.GetFiles(cachedDirectory)
.Select(found => VersionInfo.Parse(Path.GetFileName(found)))
.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true));
// NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to
// only contain versions that meet the module version constraint; for shared modules, we want to
// keep the latest version that is compatible with the tool and is equal-or-greater than the
// module version constraint.
if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null))
{
foundCached = foundCached.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList();
}
// attempt to identify the newest module version compatible with the tool
ModuleManifest manifest = null;
var match = VersionInfo.FindLatestMatchingVersion(foundCached, moduleInfo.Version, candidate => {
var candidateManifestText = File.ReadAllText(Path.Combine(Settings.GetOriginCacheDirectory(moduleInfo), candidate.ToString()));
manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText);
// check if module is compatible with this tool
return(manifest.CoreServicesVersion.IsCoreServicesCompatible(Settings.ToolVersion));
});
if (manifest != null)
{
cached = true;
// TODO (2019-10-08, bjorg): what source bucket name should be used for cached manifests?
return(MakeModuleLocation(Settings.DeploymentBucketName, manifest));
}
}
// check if module can be found in the deployment bucket
var result = await FindNewestModuleVersionAsync(Settings.DeploymentBucketName);
// check if the origin bucket needs to be checked
if (
allowImport &&
(Settings.DeploymentBucketName != moduleInfo.Origin) &&
(
// no version has been found
(result.Version == null)
// no module version constraint was given; the ultimate floating version
|| (moduleInfo.Version == null)
// the module version constraint is for a pre-release; we always prefer the origin version then
|| moduleInfo.Version.IsPreRelease
// the module version constraint is floating; we need to check if origin has a newer version
|| moduleInfo.Version.HasFloatingConstraints
)
)
{
var originResult = await FindNewestModuleVersionAsync(moduleInfo.Origin);
// check if module found at origin should be kept instead
if (
(originResult.Version != null) &&
(
(result.Version == null) ||
(moduleInfo.Version?.IsPreRelease ?? false) ||
originResult.Version.IsGreaterThanVersion(result.Version)
)
)
{
result = originResult;
}
}
// check if a module was found
if (result.Version == null)
{
// could not find a matching version
var versionConstraint = (moduleInfo.Version != null)
? $"v{moduleInfo.Version} or later"
: "any version";
if (showError)
{
LogError($"could not find module '{moduleInfo}' ({versionConstraint})");
}
return(null);
}
LogInfoVerbose($"... selected module {moduleInfo.WithVersion(result.Version)} from {result.Origin}");
// cache found version
Directory.CreateDirectory(cachedDirectory);
await File.WriteAllTextAsync(Path.Combine(cachedDirectory, result.Version.ToString()), JsonConvert.SerializeObject(result.Manifest));
return(MakeModuleLocation(result.Origin, result.Manifest));
} finally {
LogInfoPerformance($"ResolveInfoToLocationAsync() for {moduleInfo}", stopwatch.Elapsed, cached);
}
async Task <(string Origin, VersionInfo Version, ModuleManifest Manifest)> FindNewestModuleVersionAsync(string bucketName)
{
// enumerate versions in bucket
var found = await FindModuleVersionsAsync(bucketName);
if (!found.Any())
{
return(Origin : bucketName, Version : null, Manifest : null);
}
// NOTE (2019-08-12, bjorg): unless the module is shared, we filter the list of found versions to
// only contain versions that meet the module version constraint; for shared modules, we want to
// keep the latest version that is compatible with the tool and is equal-or-greater than the
// module version constraint.
if ((dependencyType != ModuleManifestDependencyType.Shared) && (moduleInfo.Version != null))
{
found = found.Where(version => version.MatchesConstraint(moduleInfo.Version)).ToList();
}
// attempt to identify the newest module version compatible with the tool
ModuleManifest manifest = null;
var match = VersionInfo.FindLatestMatchingVersion(found, moduleInfo.Version, candidate => {
var candidateModuleInfo = new ModuleInfo(moduleInfo.Namespace, moduleInfo.Name, candidate, moduleInfo.Origin);
var candidateManifestText = GetS3ObjectContentsAsync(bucketName, candidateModuleInfo.VersionPath).Result;
manifest = JsonConvert.DeserializeObject <ModuleManifest>(candidateManifestText);
// check if module is compatible with this tool
return(manifest.CoreServicesVersion.IsCoreServicesCompatible(Settings.ToolVersion));
});
return(Origin : bucketName, Version : match, Manifest : manifest);
}
async Task <IEnumerable <VersionInfo> > FindModuleVersionsAsync(string bucketName)
{
// get bucket region specific S3 client
var s3Client = await GetS3ClientByBucketNameAsync(bucketName);
if (s3Client == null)
{
// nothing to do; GetS3ClientByBucketName already emitted an error
return(new List <VersionInfo>());
}
// enumerate versions in bucket
var versions = new List <VersionInfo>();
var request = new ListObjectsV2Request {
BucketName = bucketName,
Prefix = $"{moduleInfo.Origin ?? Settings.DeploymentBucketName}/{moduleInfo.Namespace}/{moduleInfo.Name}/",
Delimiter = "/",
MaxKeys = 100,
RequestPayer = RequestPayer.Requester
};
do
{
try {
var response = await s3Client.ListObjectsV2Async(request);
versions.AddRange(response.S3Objects
.Select(s3Object => s3Object.Key.Substring(request.Prefix.Length))
.Select(found => VersionInfo.Parse(found))
.Where(version => (moduleInfo.Version == null) || version.IsGreaterOrEqualThanVersion(moduleInfo.Version, strict: true))
);
request.ContinuationToken = response.NextContinuationToken;
} catch (AmazonS3Exception e) when(e.Message == "Access Denied")
{
break;
}
} while(request.ContinuationToken != null);
LogInfoVerbose($"... found {versions.Count} version{((versions.Count == 1) ? "" : "s")} in {bucketName} [{s3Client.Config.RegionEndpoint.SystemName}]");
return(versions);
}
ModuleLocation MakeModuleLocation(string sourceBucketName, ModuleManifest manifest)
=> new ModuleLocation(sourceBucketName, manifest.ModuleInfo, manifest.TemplateChecksum);
}
public async Task <IActionResult> OnPostAsync(int?id, int?coid)
{
if (id == null)
{
return(Redirect(Url.Content("~/Forbidden")));;
}
Contest = await _context.Contest.FindAsync(id);
Settings = await _context.Settings.FirstOrDefaultAsync(m => m.CONTESTID == id);
Questions = await _context.Questions.FirstOrDefaultAsync(m => m.CONTESTID == id);
Mails = await _context.Mails.FirstOrDefaultAsync(m => m.CONTESTID == id);
Company = await _context.Company.FirstOrDefaultAsync(m => m.Id == coid);
AnswersList = await _context.Answers.Where(x => x.CONTESTID == Contest.ID).ToListAsync();
PlayerList = await _context.Player.Where(x => x.CONTESTID == Contest.ID).ToListAsync();
if (Contest != null)
{
await DeleteFile();
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = _bucketName,
Prefix = _context.Company.FirstOrDefaultAsync(m => m.Id == _userManager.GetUserAsync(User).Result.CompanyId).Result.CompanyName + "/" + Contest.NAME + "/"
};
ListObjectsV2Response response;
do
{
response = await _s3Client.ListObjectsV2Async(request);
// Process the response.
foreach (S3Object entry in response.S3Objects)
{
await FileHandler.DeleteFileS3(entry.Key);
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
foreach (var item in AnswersList)
{
_context.Answers.Remove(item);
}
foreach (var item in PlayerList)
{
_context.Player.Remove(item);
}
_context.Contest.Remove(Contest);
_context.Settings.Remove(Settings);
_context.Questions.Remove(Questions);
_context.Mails.Remove(Mails);
await _context.SaveChangesAsync();
}
return(Redirect(Url.Content("~/Contests/") + Company.Id));
}
