public AmazonS3UploadFile()
{
/* Environment Variables to set (sample):
* "AWS_ACCESS_KEY_ID": "--AKIA5J71TNDA53PISACAUXJ",
* "AWS_SECRET_ACCESS_KEY": "KaPS/59PPOSUGm1SowWxu0iQFL5jPhJcBXZAPXQoQ89Vt",
*/
var AwsBucketName = Environment.GetEnvironmentVariable("AWS_BUCKET_NAME");
var hasErrors = new List <string>();
if (AwsBucketName == null)
{
hasErrors.Add($"{nameof(AwsBucketName)} variável de ambiente não pode ser nula");
}
if (hasErrors.Count > 0)
{
throw new InternalServerError(string.Join(" | ", hasErrors.ToArray()));
}
_AwsBucketName = AwsBucketName;
var credentials = new EnvironmentVariablesAWSCredentials();
Amazon.RegionEndpoint region = Amazon.RegionEndpoint.USEast1;
_s3Client = new AmazonS3Client(credentials, region);
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
services.BindConfigurationToContainer <KafkaProducerConfiguration>(Configuration, "Kafka");
services.BindConfigurationToContainer <DataLakeWriterConfiguration>(Configuration, "Storage");
services.BindConfigurationToContainer <FileTypeValidationConfiguration>(Configuration, "Validation");
services.BindConfigurationToContainer <AmazonS3Config>(Configuration, "AmazonGeneralConfig");
services.BindConfigurationToContainer <AmazonRootIndexFileConfiguration>(Configuration, "AmazonRootIndex");
services.BindConfigurationToContainer <RedisIndexCacheConfiguration>(Configuration, "RedisIndexCache");
services.BindConfigurationToContainer <AmazonRootIndexFileConfiguration>(Configuration, "AmazonRootIndex");
services.BindConfigurationToContainer <BasicAmazonIndexFileConfiguration>(Configuration, "AmazonIndexFile");
var redisConnectionString = Configuration.GetValue <string>("Redis");
var redisConnection = ConnectionMultiplexer.Connect(redisConnectionString);
services.AddSingleton <IConnectionMultiplexer>(redisConnection);
var awsCredentials = new EnvironmentVariablesAWSCredentials();
services.AddSingleton <AWSCredentials>(awsCredentials);
services.AddSingleton <IRootIndexAccess, AmazonRootIndexFileAccess>();
services.AddSingleton <IRootIndexCache, RedisRootIndexCache>();
services.AddSingleton <IAmazonAdapter, AmazonClientAdapter>();
services.AddSingleton <ILock, RedisLock>();
services.AddSingleton <IQueryExecutor, NeedleInHaystackQueryExecutor>();
services.AddSingleton <IQueryExecutor, FrequencyQueryExecutor>();
services.AddSingleton <IQueryExecutor, CardinalityQueryExecutor>();
services.AddSingleton <IDataLakeWriter, AmazonDataLakeWriter>();
services.AddSingleton <IFileTypeValidator, ClosedListFileTypeValidator>();
services.AddSingleton <IProducerFactory, KafkaProducerFactory>();
services.AddSwaggerGen();
}
static async Task Main(string[] args)
{
var configuration = BuildConfiguration(args);
IServiceCollection serviceCollection = new ServiceCollection();
serviceCollection.AddLogging();
serviceCollection.BindConfigurationToContainer <KafkaConsumerConfiguration>(configuration, "Kafka");
serviceCollection.BindConfigurationToContainer <RedisIndexCacheConfiguration>(configuration, "RedisIndexCache");
serviceCollection.BindConfigurationToContainer <AmazonRootIndexFileConfiguration>(configuration, "AmazonRootIndex");
serviceCollection.BindConfigurationToContainer <BasicAmazonIndexFileConfiguration>(configuration, "AmazonIndexFile");
serviceCollection.BindConfigurationToContainer <NeedleInHaystackIndexConfiguration>(configuration, "NeedleInHaystackIndex");
serviceCollection.BindConfigurationToContainer <AmazonS3Config>(configuration, "AmazonGeneralConfig");
var redisConnectionString = configuration.GetValue <string>("Redis");
var redisConnection = await ConnectionMultiplexer.ConnectAsync(redisConnectionString);
serviceCollection.AddSingleton <IConnectionMultiplexer>(redisConnection);
var awsCredentials = new EnvironmentVariablesAWSCredentials();
serviceCollection.AddSingleton <AWSCredentials>(awsCredentials);
serviceCollection.AddSingleton <IConsumerFactory, KafkaConsumerFactory>();
serviceCollection.AddSingleton <IOrchestrator, ContentKafkaOrchestrator>();
serviceCollection.AddSingleton <IIndexFileWriter, AmazonIndexFileWriter>();
serviceCollection.AddSingleton <IFileTableWrapperFactory, CsvFileTableWrapperFactory>();
serviceCollection.AddSingleton <IContentIndexer, NeedleInHaystackContentIndexer>();
serviceCollection.AddSingleton <IIndexFileAccess, NeedleInHaystackIndexFileAccess>();
serviceCollection.AddSingleton <IIndexFileWriter, AmazonIndexFileWriter>();
serviceCollection.AddSingleton <IRootIndexAccess, AmazonRootIndexFileAccess>();
serviceCollection.AddSingleton <IRootIndexCache, RedisRootIndexCache>();
serviceCollection.AddSingleton <IAmazonAdapter, AmazonClientAdapter>();
serviceCollection.AddSingleton <ILock, RedisLock>();
var serviceProvider = serviceCollection.BuildServiceProvider();
var orchestrator = serviceProvider.GetService <IOrchestrator>();
await orchestrator.StartOrchestration();
}
public void EnvironmentalVariablesAWSCredentialsTest()
{
SetEnvironmentVariable("testKeyId", "testSecretAccessKey", "testSessionToken");
EnvironmentVariablesAWSCredentials EnvCredentials = new EnvironmentVariablesAWSCredentials();
Assert.AreEqual(EnvCredentials.GetCredentials(), new ImmutableCredentials("testKeyId", "testSecretAccessKey", "testSessionToken"));
}
public void SetUp()
{
var credentials = new EnvironmentVariablesAWSCredentials();
sqsClient = new AmazonSQSClient(credentials);
snsClient = new AmazonSimpleNotificationServiceClient(credentials);
s3Client = new AmazonS3Client(credentials);
}
/// <exception cref="AggregateException">
/// If both InstanceProfile and EnvironmentVariable Credentials fail.
/// Contains AmazonClientExceptions for both InstanceProfile and EnvironmentVariable failures</exception>
/// <exception cref="AmazonClientException">If Basic (Account) Credentials fail</exception>
public bool TryGetCredentials(ILog log, out AWSCredentials credentials)
{
credentials = null;
if (Credentials.Type == "account")
{
try
{
credentials = new BasicAWSCredentials(Credentials.Account.AccessKey, Credentials.Account.SecretKey);
}
// Catching a generic Exception because AWS SDK throws undocumented exceptions.
catch (Exception e)
{
log.Warn("Unable to authorise credentials, see verbose log for details.");
log.Verbose($"Unable to authorise credentials for Account: {e}");
return(false);
}
}
else
{
try
{
// If not currently running on an EC2 instance,
// this will throw an exception.
credentials = new InstanceProfileAWSCredentials();
}
// Catching a generic Exception because AWS SDK throws undocumented exceptions.
catch (Exception instanceProfileException)
{
try
{
// The last attempt is trying to use Environment Variables.
credentials = new EnvironmentVariablesAWSCredentials();
}
// Catching a generic Exception because AWS SDK throws undocumented exceptions.
catch (Exception environmentVariablesException)
{
log.Warn("Unable to authorise credentials, see verbose log for details.");
log.Verbose($"Unable to authorise credentials for Instance Profile: {instanceProfileException}");
log.Verbose($"Unable to authorise credentials for Environment Variables: {environmentVariablesException}");
return(false);
}
}
}
if (Role.Type == "assumeRole")
{
credentials = new AssumeRoleAWSCredentials(credentials,
Role.Arn,
Role.SessionName ?? DefaultSessionName,
new AssumeRoleAWSCredentialsOptions
{
ExternalId = Role.ExternalId, DurationSeconds = Role.SessionDuration
});
}
return(true);
}
public BaseRepository()
{
AWSCredentials credentials = new EnvironmentVariablesAWSCredentials();
AmazonS3Config config = new AmazonS3Config()
{
RegionEndpoint = RegionEndpoint.GetBySystemName(System.Environment.GetEnvironmentVariable("REGION"))
};
_s3Client = new AmazonS3Client(credentials, config);
}
public BaseRepository()
{
AWSCredentials credential = new EnvironmentVariablesAWSCredentials();
var config = new AmazonDynamoDBConfig()
{
RegionEndpoint = RegionEndpoint.GetBySystemName(System.Environment.GetEnvironmentVariable("REGION"))
};
_client = new AmazonDynamoDBClient(credential, config);
_Context = new DynamoDBContext(_client);
}
public BaseNotification(string topicArn)
{
AWSCredentials credentials = new EnvironmentVariablesAWSCredentials();
var config = new AmazonSimpleNotificationServiceConfig()
{
RegionEndpoint = RegionEndpoint.GetBySystemName(System.Environment.GetEnvironmentVariable("REGION"))
};
_client = new AmazonSimpleNotificationServiceClient(config);
_topicArn = topicArn;
}
public async System.Threading.Tasks.Task SnsTest()
{
var credentials = new EnvironmentVariablesAWSCredentials();
var client = new AmazonSimpleNotificationServiceClient(credentials, RegionEndpoint.APSouth1);
//SendMessage(client).Wait();
var request = new PublishRequest
{
TopicArn = "arn:aws:sns:ap-south-1:722160623806:Request-Requisition",
Message = "New Request Requisition Added by tahanee"
};
var response = await client.PublishAsync(request);
Assert.Equal(HttpStatusCode.OK, response.HttpStatusCode);
}
public IActionResult Create(ResourceRequisition resourceRequisition)
{
var credentials = new EnvironmentVariablesAWSCredentials();
if (resourceRequisition.Id == null)
{
Guid obj = Guid.NewGuid();
string gid = obj.ToString();
resourceRequisition.Id = gid;
}
_context.ResourceRequisitions.Add(resourceRequisition);
_context.SaveChanges();
var client = new AmazonSimpleNotificationServiceClient(credentials, RegionEndpoint.APSouth1);
SendMessage(client, resourceRequisition).Wait();
return(CreatedAtRoute("GetToJobs", new { id = resourceRequisition.Id }, resourceRequisition));
}
private static AWSCredentials GetCredentialSourceAWSCredentials(string credentialSourceType, bool throwIfInvalid)
{
AWSCredentials credentials;
CredentialSourceType type;
try
{
type = (CredentialSourceType)Enum.Parse(typeof(CredentialSourceType), credentialSourceType, true);
}
catch
{
return(ThrowOrReturnNull(string.Format(CultureInfo.InvariantCulture,
"Credential source [{0}] is invalid.", credentialSourceType), null, throwIfInvalid));
}
switch (type)
{
case CredentialSourceType.Ec2InstanceMetadata:
credentials = DefaultInstanceProfileAWSCredentials.Instance;
break;
case CredentialSourceType.Environment:
credentials = new EnvironmentVariablesAWSCredentials();
break;
case CredentialSourceType.EcsContainer:
var relativeUri = Environment.GetEnvironmentVariable(ECSTaskCredentials.ContainerCredentialsURIEnvVariable);
var fullUri = Environment.GetEnvironmentVariable(ECSTaskCredentials.ContainerCredentialsFullURIEnvVariable);
if (string.IsNullOrEmpty(relativeUri) && string.IsNullOrEmpty(fullUri))
{
return(ThrowOrReturnNull($"Cannot fetch credentials from container - neither {ECSTaskCredentials.ContainerCredentialsURIEnvVariable} or {ECSTaskCredentials.ContainerCredentialsFullURIEnvVariable}" +
" environment variables are set.", null, throwIfInvalid));
}
credentials = new ECSTaskCredentials(null);
break;
default:
return(ThrowOrReturnNull(string.Format(CultureInfo.InvariantCulture,
"Credential source [{0}] is not implemented.", credentialSourceType), null, throwIfInvalid));
}
return(credentials);
}
public async Task <ActionResult <List <MyCandidates> > > GetAllCandidatesAsync()
{
try{
ReceiveMessageRequest request;
var credentials = new EnvironmentVariablesAWSCredentials();
var client = new AmazonSQSClient(credentials, RegionEndpoint.APSouth1);
request = new ReceiveMessageRequest
{
AttributeNames = new List <string>()
{
"All"
},
MaxNumberOfMessages = 5,
QueueUrl = "https://sqs.ap-south-1.amazonaws.com/722160623806/candidateQueue",
VisibilityTimeout = (int)TimeSpan.FromMinutes(10).TotalSeconds,
WaitTimeSeconds = (int)TimeSpan.FromSeconds(5).TotalSeconds
};
ReceiveMessageResponse response = await client.ReceiveMessageAsync(request);
if
(response != null)
{
foreach (var message in response.Messages)
{
Console.WriteLine("For message ID '" + message.MessageId + "':");
Console.WriteLine(" Response: " + message.Body);
Console.WriteLine(" MD5Signature: " + message.MD5OfBody);
foreach (var attr in message.Attributes)
{
Console.WriteLine(" " + attr.Key + ": " + attr.Value);
}
}
}
else
{
Console.WriteLine("No messages received.");
}
}
catch (Exception ex)
{
Console.WriteLine("AWS ERROR : " + ex.Message);
}
return(_context.MyCandidates.ToList());
}
public static void Main(string[] args)
{
try
{
string myQueueURL = "https://sqs.eu-west-1.amazonaws.com/028670304332/etiquetaworten-create";
//AmazonSQSConfig sqsConfig = new AmazonSQSConfig();
//sqsConfig.ServiceURL = "http://sqs.eu-west-1.amazonaws.com";
Environment.SetEnvironmentVariable("AWS_ACCESS_KEY_ID", "<KEY_ID>");
Environment.SetEnvironmentVariable("AWS_SECRET_ACCESS_KEY", "<ACCESS_KEY>");
Environment.SetEnvironmentVariable("AWS_REGION", "eu-west-1");
var x = new EnvironmentVariablesAWSCredentials();
AmazonSQSClient sqsClient = new AmazonSQSClient(x);
SendMessageRequest sendMessageRequest = new SendMessageRequest();
sendMessageRequest.QueueUrl = myQueueURL;
sendMessageRequest.MessageBody = "Hello World!";
SendMessageResponse sendMessageResponse = sqsClient.SendMessage(sendMessageRequest);
if (sendMessageResponse.HttpStatusCode.Equals(System.Net.HttpStatusCode.OK))
{
// custom logic on success
}
else
{
// custom logic on Error
}
Console.WriteLine(sendMessageResponse.HttpStatusCode);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
public async System.Threading.Tasks.Task SqsTest()
{
var credentials = new EnvironmentVariablesAWSCredentials();
var client = new AmazonSQSClient(credentials, RegionEndpoint.APSouth1);
ReceiveMessageRequest request;
request = new ReceiveMessageRequest
{
AttributeNames = new List <string>()
{
"All"
},
MaxNumberOfMessages = 5,
QueueUrl = "https://sqs.ap-south-1.amazonaws.com/722160623806/candidateQueue",
VisibilityTimeout = (int)TimeSpan.FromMinutes(10).TotalSeconds,
WaitTimeSeconds = (int)TimeSpan.FromSeconds(5).TotalSeconds
};
ReceiveMessageResponse response = await client.ReceiveMessageAsync(request);
Assert.Equal(HttpStatusCode.OK, response.HttpStatusCode);
}
/// <summary>
/// Resolves the set of <see cref="AWSCredentials">AWS Credentials</see> based on the
/// combination of credential-related parameters that are specified.
/// </summary>
/// <remarks>
/// The order of resolution is as follows:
/// <list>
/// <item>
/// 1. If AccessKeyId is found
/// <item>a. If Session Token is found, returns Session AWS Credential</item>
/// <item>b. If no Session Token, returns a Base AWS Credential</item>
/// </item>
/// <item>
/// 2. If Profile Name is found, return a Stored Profile AWS Credential, with
/// an optional, overridden Profile Location
/// </item>
/// <item>
/// 3. If an IAM Role Name is specified, get the credentials from the local
/// EC2 instance IAM Role environment; if the special name '*' is used,
/// it uses the first IAM Role found in the current EC2 environment
/// </item>
/// <item>
/// 4. Otherwise, assume credentials are specified in environment variables
/// accessible to the hosting process and retrieve them from the following
/// variables:
/// <item><code>AWS_ACCESS_KEY_ID</code></item>
/// <item><code>AWS_SECRET_ACCESS_KEY</code></item>
/// <item><code></code>AWS_SESSION_TOKEN</code> (optional)</code></item>
/// </item>
/// </list>
/// </remarks>
public AWSCredentials ResolveCredentials()
{
AWSCredentials cr;
if (!string.IsNullOrEmpty(AwsAccessKeyId))
{
if (!string.IsNullOrEmpty(AwsSessionToken))
{
cr = new SessionAWSCredentials(AwsAccessKeyId, AwsSecretAccessKey, AwsSessionToken);
}
else
{
cr = new Amazon.Runtime.BasicAWSCredentials(AwsAccessKeyId, AwsSecretAccessKey);
}
}
else if (!string.IsNullOrEmpty(AwsProfileName))
{
cr = new StoredProfileAWSCredentials(AwsProfileName, AwsProfileLocation);
}
else if (!string.IsNullOrEmpty(AwsIamRole))
{
if (AwsIamRole == IAM_ROLE_ANY)
{
cr = new InstanceProfileAWSCredentials();
}
else
{
cr = new InstanceProfileAWSCredentials(AwsIamRole);
}
}
else
{
cr = new EnvironmentVariablesAWSCredentials();
}
return(cr);
}
public static bool TryGetCredentials(
this IAWSCredentialsArguments self,
PSHost psHost,
out AWSPSCredentials credentials,
SessionState sessionState)
{
if (self == null)
{
throw new ArgumentNullException("self");
}
credentials = null;
string name = null;
var source = CredentialsSource.Unknown;
var userSpecifiedProfile = !string.IsNullOrEmpty(self.ProfileName);
var profileChain = new CredentialProfileStoreChain(self.ProfileLocation);
// we probe for credentials by first checking the bound parameters to see if explicit credentials
// were supplied (keys, profile name, credential object), overriding anything in the shell environment
if (AWSCredentialsFactory.TryGetAWSCredentials(
self.GetCredentialProfileOptions(),
profileChain,
out var innerCredentials))
{
source = CredentialsSource.Strings;
name = "Supplied Key Parameters";
SetProxyAndCallbackIfNecessary(innerCredentials, self, psHost, sessionState);
}
// user gave us the profile name?
if (innerCredentials == null && userSpecifiedProfile)
{
if (profileChain.TryGetProfile(self.ProfileName, out var credentialProfile))
{
innerCredentials = AWSCredentialsFactory.GetAWSCredentials(credentialProfile, profileChain);
source = CredentialsSource.Profile;
name = self.ProfileName;
SetProxyAndCallbackIfNecessary(innerCredentials, self, psHost, sessionState);
}
else
{
// if the user gave us an explicit profile name (and optional location) it's an error if we
// don't find it as otherwise we could drop through and pick up a 'default' profile that is
// for a different account
return(false);
}
}
// how about an aws credentials object?
if (innerCredentials == null && self.Credential != null)
{
innerCredentials = self.Credential;
source = CredentialsSource.CredentialsObject;
name = "Credentials Object";
// don't set proxy and callback, use self.Credential as-is
}
// shell session variable set (this allows override of machine-wide environment variables)
if (innerCredentials == null && sessionState != null)
{
if (TryGetAWSPSCredentialsFromConflictingType(
sessionState.PSVariable.GetValue(SessionKeys.AWSCredentialsVariableName),
out var psCredentials))
{
credentials = psCredentials;
source = CredentialsSource.Session;
innerCredentials = credentials.Credentials; // so remaining probes are skipped
// don't set proxy and callback, use credentials.Credentials as-is
}
}
// no explicit command-level or shell instance override set, start to inspect the environment
// starting environment variables
if (innerCredentials == null)
{
try
{
var environmentCredentials = new EnvironmentVariablesAWSCredentials();
innerCredentials = environmentCredentials;
source = CredentialsSource.Environment;
name = "Environment Variables";
// no need to set proxy and callback - only basic or session credentials
}
catch
{
}
}
// get credentials from a 'default' profile?
if (innerCredentials == null && !userSpecifiedProfile)
{
if (profileChain.TryGetProfile(SettingsStore.PSDefaultSettingName, out var credentialProfile) &&
credentialProfile.CanCreateAWSCredentials)
{
innerCredentials = AWSCredentialsFactory.GetAWSCredentials(credentialProfile, profileChain);
source = CredentialsSource.Profile;
name = SettingsStore.PSDefaultSettingName;
SetProxyAndCallbackIfNecessary(innerCredentials, self, psHost, sessionState);
}
}
// get credentials from a legacy default profile name?
if (innerCredentials == null)
{
if (profileChain.TryGetProfile(SettingsStore.PSLegacyDefaultSettingName, out var credentialProfile) &&
credentialProfile.CanCreateAWSCredentials)
{
if (AWSCredentialsFactory.TryGetAWSCredentials(
credentialProfile,
profileChain,
out innerCredentials))
{
source = CredentialsSource.Profile;
name = SettingsStore.PSLegacyDefaultSettingName;
SetProxyAndCallbackIfNecessary(innerCredentials, self, psHost, sessionState);
}
}
}
if (innerCredentials == null)
{
// try and load credentials from ECS endpoint (if the relevant environment variable is set)
// or EC2 Instance Profile as a last resort
try
{
var relativeUri =
Environment.GetEnvironmentVariable(ECSTaskCredentials.ContainerCredentialsURIEnvVariable);
var fullUri = Environment.GetEnvironmentVariable(
ECSTaskCredentials.ContainerCredentialsFullURIEnvVariable);
if (!string.IsNullOrEmpty(relativeUri) || !string.IsNullOrEmpty(fullUri))
{
innerCredentials = new ECSTaskCredentials();
source = CredentialsSource.Container;
name = "Container";
// no need to set proxy and callback
}
else
{
innerCredentials = new InstanceProfileAWSCredentials();
source = CredentialsSource.InstanceProfile;
name = "Instance Profile";
// no need to set proxy and callback
}
}
catch
{
innerCredentials = null;
}
}
if (credentials == null && innerCredentials != null)
{
credentials = new AWSPSCredentials(innerCredentials, name, source);
}
return(credentials != null);
}
static IAmazonSimpleNotificationService CreateSnsClient()
{
var credentials = new EnvironmentVariablesAWSCredentials();
return(new AmazonSimpleNotificationServiceClient(credentials));
}
static IAmazonS3 CreateS3Client()
{
var credentials = new EnvironmentVariablesAWSCredentials();
return(new AmazonS3Client(credentials));
}
//@ TODO : Need to find a way for DI for AmazonSimpleNotificationServiceClient
public SNSController()
{
_AwsCredentials = new EnvironmentVariablesAWSCredentials();
}
