/// <summary>
/// Start the service, and stop the service if there were any errors found.
/// </summary>
/// <param name="Arguments">Command line arguments (unused).</param>
protected override void OnStart(string[] Arguments)
{
// Create a log file for any start-up messages
Log = new LogWriter("CrashReportProcess", LogFolder);
Config.LoadConfig();
Slack = new SlackWriter
{
WebhookUrl = Config.Default.SlackWebhookUrl,
Channel = Config.Default.SlackChannel,
Username = Config.Default.SlackUsername,
IconEmoji = Config.Default.SlackEmoji
};
Symbolicator = new Symbolicator();
StatusReporter = new StatusReporting();
ReportIndex = new ReportIndex
{
IsEnabled = !string.IsNullOrWhiteSpace(Config.Default.ProcessedReportsIndexPath),
Filepath = Config.Default.ProcessedReportsIndexPath,
Retention = TimeSpan.FromDays(Config.Default.ReportsIndexRetentionDays)
};
ReportIndex.ReadFromFile();
WriteEvent("Initializing AWS");
string AWSError;
AWSCredentials AWSCredentialsForDataRouter = new StoredProfileAWSCredentials(Config.Default.AWSProfileInputName, Config.Default.AWSCredentialsFilepath);
AmazonSQSConfig SqsConfigForDataRouter = new AmazonSQSConfig
{
ServiceURL = Config.Default.AWSSQSServiceInputURL
};
AmazonS3Config S3ConfigForDataRouter = new AmazonS3Config
{
ServiceURL = Config.Default.AWSS3ServiceInputURL
};
DataRouterAWS = new AmazonClient(AWSCredentialsForDataRouter, SqsConfigForDataRouter, S3ConfigForDataRouter, out AWSError);
if (!DataRouterAWS.IsSQSValid || !DataRouterAWS.IsS3Valid)
{
WriteFailure("AWS failed to initialize profile for DataRouter access. Error:" + AWSError);
StatusReporter.Alert("AWSFailInput", "AWS failed to initialize profile for DataRouter access", 0);
}
AWSCredentials AWSCredentialsForOutput = new StoredProfileAWSCredentials(Config.Default.AWSProfileOutputName, Config.Default.AWSCredentialsFilepath);
AmazonS3Config S3ConfigForOutput = new AmazonS3Config
{
ServiceURL = Config.Default.AWSS3ServiceOutputURL
};
OutputAWS = new AmazonClient(AWSCredentialsForOutput, null, S3ConfigForOutput, out AWSError);
if (!OutputAWS.IsS3Valid)
{
WriteFailure("AWS failed to initialize profile for output S3 bucket access. Error:" + AWSError);
StatusReporter.Alert("AWSFailOutput", "AWS failed to initialize profile for output S3 bucket access", 0);
}
// Add directory watchers
WriteEvent("Creating ReportWatcher");
Watcher = new ReportWatcher();
WriteEvent("Creating ReportProcessors");
for (int ProcessorIndex = 0; ProcessorIndex < Config.Default.ProcessorThreadCount; ProcessorIndex++)
{
var Processor = new ReportProcessor(Watcher, ProcessorIndex);
Processors.Add(Processor);
}
// Init events by enumerating event names
WriteEvent("Initializing Event Counters");
FieldInfo[] EventNameFields = typeof(StatusReportingEventNames).GetFields(BindingFlags.Static | BindingFlags.Public);
StatusReporter.InitCounters(EventNameFields.Select(EventNameField => (string)EventNameField.GetValue(null)));
WriteEvent("Initializing Folder Monitors");
Dictionary <string, string> FoldersToMonitor = new Dictionary <string, string>();
FoldersToMonitor.Add(Config.Default.ProcessedReports, "Processed Reports");
FoldersToMonitor.Add(Config.Default.ProcessedVideos, "Processed Videos");
FoldersToMonitor.Add(Config.Default.DepotRoot, "P4 Workspace");
FoldersToMonitor.Add(Config.Default.InternalLandingZone, "CRR Landing Zone");
FoldersToMonitor.Add(Config.Default.DataRouterLandingZone, "Data Router Landing Zone");
FoldersToMonitor.Add(Config.Default.PS4LandingZone, "PS4 Landing Zone");
FoldersToMonitor.Add(Config.Default.InvalidReportsDirectory, "Invalid Reports");
FoldersToMonitor.Add(Assembly.GetExecutingAssembly().Location, "CRP Binaries and Logs");
FoldersToMonitor.Add(Config.Default.MDDPDBCachePath, "MDD PDB Cache");
StatusReporter.InitFolderMonitors(FoldersToMonitor);
WriteEvent("Starting StatusReporter");
StatusReporter.Start();
// Start the threads now
Watcher.Start();
foreach (var Processor in Processors)
{
Processor.Start();
}
DateTime StartupTime = DateTime.UtcNow;
WriteEvent("Successfully started at " + StartupTime);
}
public MockS3Client(AWSCredentials credentials, AmazonS3Config clientConfig)
: base(credentials, clientConfig)
{
}
public static void Initialize(TestContext a)
{
var config = new AmazonS3Config();
s3Client = new AmazonS3Client(config);
}
public bool sendMyFileToS3(string localFilePath, string bucketName, string subDirectoryInBucket, string fileNameInS3, string s3ServiceUrl, string awsaccess, string awssecret, out string info, out string fullBucket)
{
info = string.Empty;
fullBucket = string.Empty;
AmazonS3Config config = new AmazonS3Config();
config.ServiceURL = s3ServiceUrl;
RegionEndpoint regionEndpoint = RegionEndpoint.USEast1;
IAmazonS3 client = new AmazonS3Client(awsaccess, awssecret, regionEndpoint);
TransferUtility utility = new TransferUtility(client);
TransferUtilityUploadRequest request = new TransferUtilityUploadRequest();
try
{
if (string.IsNullOrWhiteSpace(subDirectoryInBucket))
{
request.BucketName = bucketName;
}
else
{
if (!bucketName.EndsWith("/"))
{
bucketName += "/";
}
request.BucketName = bucketName + subDirectoryInBucket;
}
if (request.BucketName.EndsWith("/"))
{
request.BucketName = request.BucketName.Substring(0, request.BucketName.Length - 1);
}
request.Key = fileNameInS3;
request.FilePath = localFilePath;
request.StorageClass = S3StorageClass.Standard;
request.CannedACL = S3CannedACL.PublicRead;
request.AutoCloseStream = true;
try
{
utility.Upload(request);
}
catch (Exception e)
{
throw new Exception(fileNameInS3 + "\n" + e.ToString());
}
info = request.Key + Environment.NewLine + request.FilePath + Environment.NewLine + request.BucketName;
IEnumerator keys = request.Metadata.Keys.GetEnumerator();
for (int x = 0; x < request.Metadata.Keys.Count; x++)
{
info += request.Metadata[keys.Current.ToString()];
keys.MoveNext();
}
fullBucket = request.BucketName + "/" + fileNameInS3;
}
catch (Exception e)
{
return(false);
}
finally
{
utility.Dispose();
client.Dispose();
config = null;
request = null;
}
return(true);
}
public override UploadResult Upload(Stream stream, string fileName)
{
var validationErrors = new List <string>();
if (string.IsNullOrEmpty(s3Settings.AccessKeyID))
{
validationErrors.Add("'Access Key' must not be empty.");
}
if (string.IsNullOrEmpty(s3Settings.SecretAccessKey))
{
validationErrors.Add("'Secret Access Key' must not be empty.");
}
if (string.IsNullOrEmpty(s3Settings.Bucket))
{
validationErrors.Add("'Bucket' must not be empty.");
}
if (GetCurrentRegion(s3Settings) == UnknownEndpoint)
{
validationErrors.Add("Please select an endpoint.");
}
if (validationErrors.Any())
{
return(new UploadResult {
Errors = validationErrors
});
}
var region = GetCurrentRegion(s3Settings);
var s3ClientConfig = new AmazonS3Config();
if (region.AmazonRegion == null)
{
s3ClientConfig.ServiceURL = "https://" + region.Hostname;
}
else
{
s3ClientConfig.RegionEndpoint = region.AmazonRegion;
}
using (var client = new AmazonS3Client(GetCurrentCredentials(), s3ClientConfig))
{
var putRequest = new GetPreSignedUrlRequest
{
BucketName = s3Settings.Bucket,
Key = GetObjectKey(fileName),
Verb = HttpVerb.PUT,
Expires = DateTime.UtcNow.AddMinutes(5),
ContentType = Helpers.GetMimeType(fileName)
};
var requestHeaders = new NameValueCollection();
requestHeaders["x-amz-acl"] = "public-read";
requestHeaders["x-amz-storage-class"] = GetObjectStorageClass();
putRequest.Headers["x-amz-acl"] = "public-read";
putRequest.Headers["x-amz-storage-class"] = GetObjectStorageClass();
var responseHeaders = SendRequestStreamGetHeaders(client.GetPreSignedURL(putRequest), stream, Helpers.GetMimeType(fileName), requestHeaders, method: HttpMethod.PUT);
if (responseHeaders.Count == 0)
{
return(new UploadResult {
Errors = new List <string> {
"Upload to Amazon S3 failed. Check your access credentials."
}
});
}
var eTag = responseHeaders.Get("ETag");
if (eTag == null)
{
return(new UploadResult {
Errors = new List <string> {
"Upload to Amazon S3 failed."
}
});
}
if (GetMd5Hash(stream) == eTag.Replace("\"", ""))
{
return(new UploadResult {
IsSuccess = true, URL = GetObjectURL(putRequest.Key)
});
}
return(new UploadResult {
Errors = new List <string> {
"Upload to Amazon S3 failed, uploaded data did not match."
}
});
}
}
public List <FileListModel> GetDirectoryList(string directory, string accessToken)
{
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USWest1, // MUST set this before setting ServiceURL and it should match the `MINIO_REGION` enviroment variable.
ForcePathStyle = true // MUST be true to work correctly with Minio server
};
string region = accessToken.Split("-")[2];
if (region == "USWest1")
{
config.RegionEndpoint = RegionEndpoint.USWest1;
}
else if (region == "USWest2")
{
config.RegionEndpoint = RegionEndpoint.USWest2;
}
else if (region == "APNortheast1")
{
config.RegionEndpoint = RegionEndpoint.APNortheast1;
}
else if (region == "APNortheast2")
{
config.RegionEndpoint = RegionEndpoint.APNortheast2;
}
else if (region == "APSouth1")
{
config.RegionEndpoint = RegionEndpoint.APSouth1;
}
else if (region == "APSoutheast1")
{
config.RegionEndpoint = RegionEndpoint.APSoutheast1;
}
else if (region == "APSoutheast2")
{
config.RegionEndpoint = RegionEndpoint.APSoutheast2;
}
else if (region == "CACentral1")
{
config.RegionEndpoint = RegionEndpoint.CACentral1;
}
else if (region == "CNNorth1")
{
config.RegionEndpoint = RegionEndpoint.CNNorth1;
}
else if (region == "CNNorthWest1")
{
config.RegionEndpoint = RegionEndpoint.CNNorthWest1;
}
else if (region == "EUCentral1")
{
config.RegionEndpoint = RegionEndpoint.EUCentral1;
}
else if (region == "EUWest2")
{
config.RegionEndpoint = RegionEndpoint.EUWest2;
}
else if (region == "EUWest3")
{
config.RegionEndpoint = RegionEndpoint.EUWest3;
}
else if (region == "SAEast1")
{
config.RegionEndpoint = RegionEndpoint.SAEast1;
}
else if (region == "USEast1")
{
config.RegionEndpoint = RegionEndpoint.USEast1;
}
else if (region == "USEast2")
{
config.RegionEndpoint = RegionEndpoint.USEast2;
}
else if (region == "EUWest1")
{
config.RegionEndpoint = RegionEndpoint.EUWest1;
}
else if (region == "USGovCloudWest1")
{
config.RegionEndpoint = RegionEndpoint.USGovCloudWest1;
}
//var amazonS3Client = new AmazonS3Client("AKIAIZ7ZNECIUB4MEKRQ", "g2OZECVv8kRsbixfiP52xnKhHo/FSny5hm0qmb4q", config);
var amazonS3Client = new AmazonS3Client(accessToken.Split("-")[0], accessToken.Split("-")[1], config);
List <FileListModel> list = new List <FileListModel>();
if (String.IsNullOrEmpty(directory))
{
var listBucketResponse = (amazonS3Client.ListBucketsAsync()).Result;
foreach (var item in listBucketResponse.Buckets)
{
list.Add(new FileListModel()
{
CloudId = item.BucketName, Directory = item.BucketName
});
}
}
else
{
string bucket = Regex.Split(directory, "/")[0];
string innerDirectory = "";
string[] directoryArray = directory.Split("/");
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = bucket,
Delimiter = @"/",
};
if (directoryArray.Length > 1)
{
for (int i = 1; i < directoryArray.Length - 1; i++)
{
innerDirectory = directoryArray[i] + "/";
}
request.Prefix = innerDirectory;
}
else
{
}
var bucketDirectory = (amazonS3Client.ListObjectsV2Async(request)).Result;
foreach (var item in bucketDirectory.CommonPrefixes)
{
list.Add(new FileListModel()
{
CloudId = bucket + "/" + item, Directory = bucket + "/" + item
});
}
}
return(list);
}
static void Configure(StandardConfigurer <ISubscriptionStorage> configurer, AWSCredentials credentials, AmazonS3Config config, AmazonS3DataBusOptions options)
{
configurer.Register(c => new AmazonS3SubscriptionsStorage(credentials, config, options));
}
public static async Task <List <s3mod> > Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
var bucketName = req.Query["bucketName"];
var access = req.Headers["access"];
var secret = req.Headers["secret"];
var region = req.Query["region"];
var prefix = req.Query["prefix"];
List <s3mod> folderContents = new List <s3mod>();
var credentials = new BasicAWSCredentials(access, secret);
var config = new AmazonS3Config
{
RegionEndpoint = andys.function.S3Region.getAWSRegion(region)
};
try
{
using var client = new AmazonS3Client(credentials, config);
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucketName,
Prefix = prefix
};
ListObjectsResponse response = await client.ListObjectsAsync(request);
foreach (S3Object o in response.S3Objects)
{
Console.WriteLine(o.Key);
folderContents.Add(new s3mod()
{
name = "Folder Item", value = o.Key
});
}
}
catch (AmazonS3Exception amazonS3Exception)
{
if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
{
Console.WriteLine("Incorrect AWS Credentials.");
folderContents.Add(new s3mod()
{
name = "Error", value = "Incorrect AWS Credentials"
});
}
else
{
Console.WriteLine("Error: ", amazonS3Exception.ErrorCode, amazonS3Exception.Message);
folderContents.Add(new s3mod()
{
name = amazonS3Exception.ErrorCode, value = amazonS3Exception.Message
});
}
}
return(folderContents);
}
static async Task Main()
{
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.MinimumLevel.Override("Microsoft", LogEventLevel.Information)
.Enrich.FromLogContext()
.WriteTo.Console()
.CreateLogger();
IConfiguration configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", true, true)
.AddEnvironmentVariables()
.Build();
var host = new HostBuilder()
.ConfigureServices((hostBuilderContext, services) =>
{
services.AddOptions();
services.Configure <NServiceBusOptions>(configuration.GetSection("NServiceBus"));
})
.UseSerilog()
.UseNServiceBus(context =>
{
var nServiceBusOptions = configuration.GetSection("NServiceBus").Get <NServiceBusOptions>();
var endpointConfiguration = new EndpointConfiguration("Samples.FullDuplex.Server");
endpointConfiguration.DoNotCreateQueues();
var amazonSqsConfig = new AmazonSQSConfig();
if (!string.IsNullOrEmpty(nServiceBusOptions.SqsServiceUrlOverride))
{
amazonSqsConfig.ServiceURL = nServiceBusOptions.SqsServiceUrlOverride;
}
var transport = endpointConfiguration.UseTransport <SqsTransport>();
transport.ClientFactory(() => new AmazonSQSClient(
new AnonymousAWSCredentials(),
amazonSqsConfig));
var amazonSimpleNotificationServiceConfig = new AmazonSimpleNotificationServiceConfig();
if (!string.IsNullOrEmpty(nServiceBusOptions.SnsServiceUrlOverride))
{
amazonSimpleNotificationServiceConfig.ServiceURL = nServiceBusOptions.SnsServiceUrlOverride;
}
transport.ClientFactory(() => new AmazonSimpleNotificationServiceClient(
new AnonymousAWSCredentials(),
amazonSimpleNotificationServiceConfig));
var amazonS3Config = new AmazonS3Config
{
ForcePathStyle = true,
};
if (!string.IsNullOrEmpty(nServiceBusOptions.S3ServiceUrlOverride))
{
amazonS3Config.ServiceURL = nServiceBusOptions.S3ServiceUrlOverride;
}
var s3Configuration = transport.S3("bucketname", "Samples-FullDuplex-Client");
s3Configuration.ClientFactory(() => new AmazonS3Client(
new AnonymousAWSCredentials(),
amazonS3Config));
endpointConfiguration.SendFailedMessagesTo("error");
endpointConfiguration.EnableInstallers();
return(endpointConfiguration);
})
.UseConsoleLifetime()
.Build();
await host.RunAsync();
}
public async Task <bool> CheckCredentialAccessAsync()
{
Dictionary <string, string> cred = devoctomy.cachy.Framework.Native.Native.PasswordVault.GetCredential(ID);
_credentialError = (cred == null);
if (!_credentialError)
{
string secret = cred["Password"];
switch (AuthType)
{
case ProviderType.AuthenticationType.OAuth:
{
Dictionary <string, string> parameters = new Dictionary <string, string>();
parameters.Add("AuthType", "OAuth");
parameters.Add("ProviderKey", ProviderKey);
parameters.Add("AccessToken", secret);
CloudStorageProviderBase provider = CloudStorageProviderBase.Create(
App.AppLogger.Logger,
parameters);
try
{
CloudProviderResponse <CloudStorageProviderUserBase> accountUserResponse = await provider.GetAccountUser();
_credentialError = !(accountUserResponse.ResponseValue == CloudProviderResponse <CloudStorageProviderUserBase> .Response.Success);
_username = String.Empty;
}
catch (Exception)
{
_credentialError = true;
}
break;
}
case ProviderType.AuthenticationType.Amazon:
{
JObject s3ConfigJSON = JObject.Parse(secret);
AmazonS3Config s3Config = AmazonS3Config.FromJSON(s3ConfigJSON);
Dictionary <string, string> parameters = s3Config.ToDictionary();
parameters.Add("ProviderKey", ProviderKey);
CloudStorageProviderBase provider = CloudStorageProviderBase.Create(
App.AppLogger.Logger,
parameters);
try
{
CloudProviderResponse <CloudStorageProviderUserBase> accountUserResponse = await provider.GetAccountUser();
_credentialError = !(accountUserResponse.ResponseValue == CloudProviderResponse <CloudStorageProviderUserBase> .Response.Success);
_username = String.Empty;
}
catch (Exception)
{
_credentialError = true;
}
break;
}
}
}
NotifyPropertyChanged("CredentialError");
NotifyPropertyChanged("UserName");
return(_credentialError);
}
public FakeMyAmazonS3ClientThatReturnsNull(AmazonS3Config config) : base(config)
{
}
public AWSS3Client(AmazonS3Config config, string accessKey, string secretKey, string bucketName, ILogger <IS3Client> logger, int retryCount, int retryInSeconds)
: this(new AmazonS3Client(awsAccessKeyId : accessKey, awsSecretAccessKey : secretKey, clientConfig : config), config.ServiceURL, bucketName, logger, retryCount, retryInSeconds)
{
}
public AWSS3Client(AmazonS3Config config, string accessKey, string secretKey, string bucketName)
: this(new AmazonS3Client(awsAccessKeyId : accessKey, awsSecretAccessKey : secretKey, clientConfig : config), config.ServiceURL, bucketName, null, 0, 0)
{
}
//[Route("[action]")]
//public async Task<ActionResult<FileVM>> PostImage(FileVM UploadedImage)
public async Task <IActionResult> PostImage(FileVM UploadedImage)
{
byte[] bytes = Convert.FromBase64String(UploadedImage.FileAsBase64);
var credentials = new BasicAWSCredentials("AKIAYFOXPUFXRIBLXF4O", "kt30oEKBt35RZRxXD6rLRd2uxITL0aYX24qFXnox");
var config = new AmazonS3Config
{
RegionEndpoint = Amazon.RegionEndpoint.USEast1
};
var image = new Image();
using (var client = new AmazonS3Client(credentials, config))
{
using (var newMemoryStream = new MemoryStream(bytes))
{
//file.CopyTo(newMemoryStream);
var uploadRequest = new TransferUtilityUploadRequest
{
InputStream = newMemoryStream,
Key = UploadedImage.FileName,
BucketName = "quickquoteitem",
CannedACL = S3CannedACL.PublicRead
};
var fileTransferUtility = new TransferUtility(client);
try
{
await fileTransferUtility.UploadAsync(uploadRequest);
}
catch (Exception err)
{
throw err;
}
}
}
AmazonRekognitionClient rekognitionClient = new AmazonRekognitionClient(credentials, Amazon.RegionEndpoint.USEast1);
DetectLabelsRequest detectlabelsRequest = new DetectLabelsRequest()
{
Image = new Image()
{
S3Object = new S3Object()
{
Name = UploadedImage.FileName,
Bucket = "quickquoteitem"
},
},
MaxLabels = 10,
MinConfidence = 75F
};
try
{
var Labels = new List <LabelVM>();
DetectLabelsResponse detectLabelsResponse =
await rekognitionClient.DetectLabelsAsync(detectlabelsRequest);
//Console.WriteLine("Detected labels for " + photo);
foreach (Label label in detectLabelsResponse.Labels)
{
var item = new LabelVM();
item.LabelName = label.Name;
item.LabelConfidence = label.Confidence.ToString();
Labels.Add(item);
}
return(Ok(Labels));
//Console.WriteLine("{0}: {1}", label.Name, label.Confidence);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
//return CreatedAtAction("GetQuote", new { id = quote.QuoteID }, quote);
return(Ok());
}
public void SetTopicConfigurationTests()
{
var s3Config = new AmazonS3Config();
using (var s3Client = new AmazonS3Client(s3Config))
using (var snsClient = new AmazonSimpleNotificationServiceClient())
using (var stsClient = new AmazonSecurityTokenServiceClient())
{
var snsCreateResponse = snsClient.CreateTopic("events-test-" + DateTime.Now.Ticks);
var bucketName = S3TestUtils.CreateBucketWithWait(s3Client);
try
{
snsClient.AuthorizeS3ToPublish(snsCreateResponse.TopicArn, bucketName);
PutBucketNotificationRequest putRequest = new PutBucketNotificationRequest
{
BucketName = bucketName,
TopicConfigurations = new List <TopicConfiguration>
{
new TopicConfiguration
{
Id = "the-topic-test",
Topic = snsCreateResponse.TopicArn,
Events = new List <EventType> {
EventType.ObjectCreatedPut
}
}
}
};
s3Client.PutBucketNotification(putRequest);
var getResponse = S3TestUtils.WaitForConsistency(() =>
{
var res = s3Client.GetBucketNotification(bucketName);
return(res.TopicConfigurations?.Count > 0 && res.TopicConfigurations[0].Id == "the-topic-test" ? res : null);
});
var getAttributeResponse = snsClient.GetTopicAttributes(new GetTopicAttributesRequest
{
TopicArn = snsCreateResponse.TopicArn
});
var policy = Policy.FromJson(getAttributeResponse.Attributes["Policy"]);
// SNS topics already have a default statement. We need to evaluate the second statement that the SDK appended.
var conditions = policy.Statements[1].Conditions;
Assert.AreEqual(2, conditions.Count);
var accountCondition = conditions.FirstOrDefault(x => string.Equals(x.ConditionKey, ConditionFactory.SOURCE_ACCOUNT_KEY));
Assert.IsNotNull(accountCondition);
Assert.AreEqual(ConditionFactory.StringComparisonType.StringEquals.ToString(), accountCondition.Type);
Assert.AreEqual(12, accountCondition.Values[0].Length);
var currentAccountId = stsClient.GetCallerIdentity(new GetCallerIdentityRequest()).Account;
Assert.AreEqual(currentAccountId, accountCondition.Values[0]);
Assert.AreEqual(1, getResponse.TopicConfigurations.Count);
Assert.AreEqual(1, getResponse.TopicConfigurations[0].Events.Count);
Assert.AreEqual(EventType.ObjectCreatedPut, getResponse.TopicConfigurations[0].Events[0]);
#pragma warning disable 618
Assert.AreEqual("s3:ObjectCreated:Put", getResponse.TopicConfigurations[0].Event);
#pragma warning restore 618
Assert.AreEqual("the-topic-test", getResponse.TopicConfigurations[0].Id);
Assert.AreEqual(snsCreateResponse.TopicArn, getResponse.TopicConfigurations[0].Topic);
}
finally
{
snsClient.DeleteTopic(snsCreateResponse.TopicArn);
AmazonS3Util.DeleteS3BucketWithObjects(s3Client, bucketName);
}
}
}
public Form1()
{
InitializeComponent();
try
{
bool tarefaExiste = false;
foreach (var item in TaskService.Instance.RootFolder.AllTasks)
{
if (item.ToString().Equals("VGestAgentTask"))
{
tarefaExiste = true;
break;
}
}
if (!tarefaExiste)
{
using (TaskService ts = new TaskService())
{
TaskDefinition td = ts.NewTaskFromFile("VGestAgentTask.xml");
ts.RootFolder.RegisterTaskDefinition("", td);
Application.Exit();
Environment.Exit(1);
}
}
}
catch (Exception)
{
Console.WriteLine("Erro na criacao da tarefa.");
}
if (Properties.Settings.Default.update)
{
checkBox_updates.Checked = true;
}
else
{
checkBox_updates.Checked = false;
}
textBox_id.Text = Properties.Settings.Default.id;
textBox_url.Text = Properties.Settings.Default.url;
notifyIcon1.ContextMenuStrip = new ContextMenuStrip();
notifyIcon1.ContextMenuStrip.Items.Add("Sair", null, Menu_Sair_Click);
AmazonS3Config config = new AmazonS3Config
{
ServiceURL = "https://fra1.digitaloceanspaces.com/"
};
BasicAWSCredentials credentials = new BasicAWSCredentials("YJ4M7GEE6N2BNPAXCTOR", "HSBr4I2eNWe4dGwRqgo+u+5wB84m7nSAK6ppO4mZ+5c");
clientAmazon = new AmazonS3Client(credentials, config);
button_get_Click(null, null);
panel_login.Visible = true;
panel_main.Visible = false;
checkVersao();
}
public AmazonAnnotationPackageProvider()
{
this._packagesToDownload = new Queue <AnnotationPackage>();
var config = this.GetConfig();
#region Validate config
if (!this.ValidateConfig(config))
{
using (var dialog = new AmazonConfigurationDialog())
{
dialog.SetConfig(config);
dialog.ShowDialog();
}
config = this.GetConfig();
if (!this.ValidateConfig(config))
{
throw new Exception("Configuration invalid");
}
}
#endregion
this._config = config;
if (string.IsNullOrEmpty(config.S3ServiceUrl) || string.IsNullOrEmpty(config.DynamoDbServiceUrl))
{
this._s3Client = new AmazonS3Client(config.AccessKeyId, config.SecretAccessKey, RegionEndpoint.EUWest1);
this._dynamoDbClient = new AmazonDynamoDBClient(config.AccessKeyId, config.SecretAccessKey, RegionEndpoint.EUWest1);
}
else
{
var s3Config = new AmazonS3Config
{
ServiceURL = config.S3ServiceUrl,
ForcePathStyle = true
};
var dbConfig = new AmazonDynamoDBConfig
{
ServiceURL = config.DynamoDbServiceUrl
};
this._s3Client = new AmazonS3Client(config.AccessKeyId, config.SecretAccessKey, s3Config);
this._dynamoDbClient = new AmazonDynamoDBClient(config.AccessKeyId, config.SecretAccessKey, dbConfig);
using (var tokenSource = new CancellationTokenSource(3000))
{
if (!this._s3Client.DoesS3BucketExistAsync(this._config.BucketName).GetAwaiter().GetResult())
{
this._s3Client.PutBucketAsync(this._config.BucketName, tokenSource.Token).GetAwaiter().GetResult();
}
var createTableRequest = new CreateTableRequest
{
TableName = this._config.DbTableName,
AttributeDefinitions = new List <AttributeDefinition>(),
KeySchema = new List <KeySchemaElement>(),
GlobalSecondaryIndexes = new List <GlobalSecondaryIndex>(),
LocalSecondaryIndexes = new List <LocalSecondaryIndex>(),
ProvisionedThroughput = new ProvisionedThroughput
{
ReadCapacityUnits = 1,
WriteCapacityUnits = 1
}
};
createTableRequest.KeySchema = new List <KeySchemaElement>
{
new KeySchemaElement
{
AttributeName = "Id",
KeyType = KeyType.HASH,
},
};
createTableRequest.AttributeDefinitions = new List <AttributeDefinition>
{
new AttributeDefinition
{
AttributeName = "Id",
AttributeType = ScalarAttributeType.S,
}
};
var tables = this._dynamoDbClient.ListTablesAsync(tokenSource.Token).GetAwaiter().GetResult();
if (!tables.TableNames.Contains(this._config.DbTableName))
{
this._dynamoDbClient.CreateTableAsync(createTableRequest, tokenSource.Token).GetAwaiter().GetResult();
}
}
}
}
public async Task <IActionResult> upload(IFormFile file, int taskId)
{
string [] permittedExtensions = { ".docx", ".pdf", ".jpg", ".png", ".xls", ".xlsx", ".ppt",
".pptx", ".txt", ".avi", ".mp4", ".mp3" };
string extension = Path.GetExtension(file.FileName).ToLower();
long fileLength = (file.Length / 1024) + 1; //find file size in kb
long maxSize = 30000;
if (_repo.getCurrentStorageUsage() < enviroment.maxStorage)
{
if (string.IsNullOrEmpty(extension) || permittedExtensions.Contains(extension) && fileLength < maxSize)
{
var credentials = new BasicAWSCredentials(enviroment.awsAccessKeyId,
enviroment.awsSecretAccessKey);
var config = new AmazonS3Config
{
RegionEndpoint = Amazon.RegionEndpoint.EUWest2
};
using var client = new AmazonS3Client(credentials, config);
try
{
await using var newMemoryStream = new MemoryStream();
file.CopyTo(newMemoryStream);
var uploadRequest = new TransferUtilityUploadRequest
{
InputStream = newMemoryStream,
Key = taskId + "/" + file.FileName,
BucketName = enviroment.bucketName,
CannedACL = S3CannedACL.PublicRead
};
var fileTransferUtility = new TransferUtility(client);
await fileTransferUtility.UploadAsync(uploadRequest);
//put the file sent in, into an attachmentFile object to be saved on the database
AttachmentFile newAttachment = new AttachmentFile();
newAttachment.FileName = file.FileName;
newAttachment.TaskScheduleId = taskId;
newAttachment.UserId = int.Parse(User.FindFirst(ClaimTypes.NameIdentifier).Value);
newAttachment.FileSize = fileLength;
DateTime thisDay = DateTime.Now;
string NowDate = thisDay.ToString("g");
newAttachment.DateCreated = Convert.ToDateTime(NowDate);
//check if attachment is already in database
if (!_repo.alreadyExist(file.FileName, taskId))
{
_repo.Add(newAttachment);
if (await _repo.SaveAll())
{
return(Ok());
}
return(BadRequest());
}
else
{
_repo.Update(newAttachment);
return(Ok());
}
}
catch (AmazonS3Exception e)
{
// If bucket or object does not exist
Console.WriteLine("Error encountered ***. Message:'{0}' when reading object", e.Message);
}
catch (Exception e)
{
Console.WriteLine("Unknown encountered on server. Message:'{0}' when reading object", e.Message);
}
}
return(BadRequest("Bad file"));
}
return(BadRequest("Storage limiation reached. Please contact the software manufacturer"));
}
private void UploadToCdn()
{
try
{
// one thread only
if (Interlocked.CompareExchange(ref work, 1, 0) == 0)
{
var @continue = false;
try
{
CdnItem item;
if (queue.TryDequeue(out item))
{
@continue = true;
var cdnpath = GetCdnPath(item.Bundle.Path);
var key = new Uri(cdnpath).PathAndQuery.TrimStart('/');
var content = Encoding.UTF8.GetBytes(item.Response.Content);
var inputStream = new MemoryStream();
if (ClientSettings.GZipEnabled)
{
using (var zip = new GZipStream(inputStream, CompressionMode.Compress, true))
{
zip.Write(content, 0, content.Length);
zip.Flush();
}
}
else
{
inputStream.Write(content, 0, content.Length);
}
var checksum = AmazonS3Util.GenerateChecksumForContent(item.Response.Content, true);
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(s3region),
UseHttp = true
};
using (var s3 = new AmazonS3Client(s3publickey, s3privatekey, config))
{
var upload = false;
try
{
var request = new GetObjectMetadataRequest
{
BucketName = s3bucket,
Key = key,
};
var response = s3.GetObjectMetadata(request);
upload = !string.Equals(checksum, response.Metadata["x-amz-meta-etag"], StringComparison.InvariantCultureIgnoreCase);
}
catch (AmazonS3Exception ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
upload = true;
}
else
{
throw;
}
}
if (upload)
{
var request = new PutObjectRequest
{
BucketName = s3bucket,
CannedACL = S3CannedACL.PublicRead,
AutoCloseStream = true,
AutoResetStreamPosition = true,
Key = key,
ContentType = AmazonS3Util.MimeTypeFromExtension(Path.GetExtension(key).ToLowerInvariant()),
InputStream = inputStream
};
if (ClientSettings.GZipEnabled)
{
request.Headers.ContentEncoding = "gzip";
}
var cache = TimeSpan.FromDays(365);
request.Headers.CacheControl = string.Format("public, maxage={0}", (int)cache.TotalSeconds);
request.Headers.ExpiresUtc = DateTime.UtcNow.Add(cache);
request.Headers["x-amz-meta-etag"] = checksum;
s3.PutObject(request);
}
else
{
inputStream.Close();
}
item.Bundle.CdnPath = cdnpath;
}
}
}
catch (Exception err)
{
log.Error(err);
}
finally
{
work = 0;
if (@continue)
{
Action upload = () => UploadToCdn();
upload.BeginInvoke(null, null);
}
}
}
}
catch (Exception fatal)
{
log.Fatal(fatal);
}
}
/// <summary>
/// Creates a new instance of <see cref="AwsS3BlobStorageProvider"/> for a given S3 client configuration/>
/// </summary>
public AwsS3BlobStorageProvider(string accessKeyId, string secretAccessKey, string bucketName, AmazonS3Config clientConfig)
{
if (accessKeyId == null)
{
throw new ArgumentNullException(nameof(accessKeyId));
}
if (secretAccessKey == null)
{
throw new ArgumentNullException(nameof(secretAccessKey));
}
_bucketName = bucketName ?? throw new ArgumentNullException(nameof(bucketName));
_client = new AmazonS3Client(new BasicAWSCredentials(accessKeyId, secretAccessKey), clientConfig);
_fileTransferUtility = new TransferUtility(_client);
}
public S3StorageBackend(TransferAgentOptions options, AWSCredentials awsCredentials, AmazonS3Config s3Config, string awsBucketName)
{
this._shouldDispose = true;
Options = options;
SanitizeOptions();
this._s3Config = s3Config;
this._s3Config.RegionEndpoint = GetBucketEndpoint(awsCredentials, awsBucketName);
this._s3Client = new AmazonS3Client(awsCredentials, s3Config);
this._awsBuckeName = awsBucketName;
}
/// <summary>
/// Parses sleet.json to find the source and constructs it.
/// </summary>
public static async Task <ISleetFileSystem> CreateFileSystemAsync(LocalSettings settings, LocalCache cache, string source)
{
ISleetFileSystem result = null;
var sources = settings.Json["sources"] as JArray;
if (sources == null)
{
throw new ArgumentException("Invalid config. No sources found.");
}
foreach (var sourceEntry in sources.Select(e => (JObject)e))
{
var sourceName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "name");
if (source.Equals(sourceName, StringComparison.OrdinalIgnoreCase))
{
var path = JsonUtility.GetValueCaseInsensitive(sourceEntry, "path");
var baseURIString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "baseURI");
var feedSubPath = JsonUtility.GetValueCaseInsensitive(sourceEntry, "feedSubPath");
var type = JsonUtility.GetValueCaseInsensitive(sourceEntry, "type")?.ToLowerInvariant();
string absolutePath;
if (path != null && type == "local")
{
if (settings.Path == null && !Path.IsPathRooted(NuGetUriUtility.GetLocalPath(path)))
{
throw new ArgumentException("Cannot use a relative 'path' without a sleet.json file.");
}
var nonEmptyPath = path == "" ? "." : path;
var absoluteSettingsPath = NuGetUriUtility.GetAbsolutePath(Directory.GetCurrentDirectory(), settings.Path);
var settingsDir = Path.GetDirectoryName(absoluteSettingsPath);
absolutePath = NuGetUriUtility.GetAbsolutePath(settingsDir, nonEmptyPath);
}
else
{
absolutePath = path;
}
var pathUri = absolutePath != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(absolutePath)) : null;
var baseUri = baseURIString != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(baseURIString)) : pathUri;
if (type == "local")
{
if (pathUri == null)
{
throw new ArgumentException("Missing path for account.");
}
result = new PhysicalFileSystem(cache, pathUri, baseUri);
}
else if (type == "azure")
{
var connectionString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "connectionString");
var container = JsonUtility.GetValueCaseInsensitive(sourceEntry, "container");
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("Missing connectionString for azure account.");
}
if (connectionString.Equals(AzureFileSystem.AzureEmptyConnectionString, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Invalid connectionString for azure account.");
}
if (string.IsNullOrEmpty(container))
{
throw new ArgumentException("Missing container for azure account.");
}
var azureAccount = CloudStorageAccount.Parse(connectionString);
if (pathUri == null)
{
// Get the default url from the container
pathUri = AzureUtility.GetContainerPath(azureAccount, container);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AzureFileSystem(cache, pathUri, baseUri, azureAccount, container, feedSubPath);
}
#if !SLEETLEGACY
else if (type == "s3")
{
var profileName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "profileName");
var accessKeyId = JsonUtility.GetValueCaseInsensitive(sourceEntry, "accessKeyId");
var secretAccessKey = JsonUtility.GetValueCaseInsensitive(sourceEntry, "secretAccessKey");
var bucketName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "bucketName");
var region = JsonUtility.GetValueCaseInsensitive(sourceEntry, "region");
var serviceURL = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serviceURL");
var serverSideEncryptionMethod = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serverSideEncryptionMethod") ?? "None";
var compress = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "compress", true);
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentException("Missing bucketName for Amazon S3 account.");
}
if (string.IsNullOrEmpty(region) && string.IsNullOrEmpty(serviceURL))
{
throw new ArgumentException("Either 'region' or 'serviceURL' must be specified for an Amazon S3 account");
}
if (!string.IsNullOrEmpty(region) && !string.IsNullOrEmpty(serviceURL))
{
throw new ArgumentException("Options 'region' and 'serviceURL' cannot be used together");
}
if (serverSideEncryptionMethod != "None" && serverSideEncryptionMethod != "AES256")
{
throw new ArgumentException("Only 'None' or 'AES256' are currently supported for serverSideEncryptionMethod");
}
AmazonS3Config config = null;
if (serviceURL != null)
{
config = new AmazonS3Config()
{
ServiceURL = serviceURL,
ProxyCredentials = CredentialCache.DefaultNetworkCredentials
};
}
else
{
config = new AmazonS3Config()
{
RegionEndpoint = RegionEndpoint.GetBySystemName(region),
ProxyCredentials = CredentialCache.DefaultNetworkCredentials
};
}
AmazonS3Client amazonS3Client = null;
// Load credentials from the current profile
if (!string.IsNullOrWhiteSpace(profileName))
{
var credFile = new SharedCredentialsFile();
if (credFile.TryGetProfile(profileName, out var profile))
{
amazonS3Client = new AmazonS3Client(profile.GetAWSCredentials(profileSource: null), config);
}
else
{
throw new ArgumentException($"The specified AWS profileName {profileName} could not be found. The feed must specify a valid profileName for an AWS credentials file. For help on credential files see: https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file");
}
}
// Load credentials explicitly with an accessKey and secretKey
else if (
!string.IsNullOrWhiteSpace(accessKeyId) &&
!string.IsNullOrWhiteSpace(secretAccessKey))
{
amazonS3Client = new AmazonS3Client(new BasicAWSCredentials(accessKeyId, secretAccessKey), config);
}
// Load credentials from Environment Variables
else if (
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(EnvironmentVariablesAWSCredentials.ENVIRONMENT_VARIABLE_ACCESSKEY)) &&
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(EnvironmentVariablesAWSCredentials.ENVIRONMENT_VARIABLE_SECRETKEY)))
{
amazonS3Client = new AmazonS3Client(new EnvironmentVariablesAWSCredentials(), config);
}
// Load credentials from an ECS docker container
else if (
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(ECSTaskCredentials.ContainerCredentialsURIEnvVariable)))
{
amazonS3Client = new AmazonS3Client(new ECSTaskCredentials(), config);
}
// Assume IAM role
else
{
using (var client = new AmazonSecurityTokenServiceClient(config.RegionEndpoint))
{
try
{
var identity = await client.GetCallerIdentityAsync(new GetCallerIdentityRequest());
}
catch (Exception ex)
{
throw new ArgumentException(
"Failed to determine AWS identity - ensure you have an IAM " +
"role set, have set up default credentials or have specified a profile/key pair.", ex);
}
}
amazonS3Client = new AmazonS3Client(config);
}
if (pathUri == null)
{
// Find the default path
pathUri = AmazonS3Utility.GetBucketPath(bucketName, config.RegionEndpoint.SystemName);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AmazonS3FileSystem(
cache,
pathUri,
baseUri,
amazonS3Client,
bucketName,
serverSideEncryptionMethod,
feedSubPath,
compress);
}
#endif
}
}
return(result);
}
/**
* Returns the connection
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
*/
public IAmazonS3 getConnection()
{
if (this.connection != null)
{
return(this.connection);
}
var scheme = _params.ContainsKey("use_ssl") && ((bool)_params["use_ssl"] == false) ? "http" : "https";
var base_url = scheme + "://" + _params["hostname"] + ":" + _params["port"] + "/";
AWSCredentials credentials = new BasicAWSCredentials((string)_params["key"], (string)_params["secret"]);
RegionEndpoint regionEndpoint = RegionEndpoint.GetBySystemName((string)_params["region"]);
AmazonS3Config amazonS3Config = new AmazonS3Config( );
amazonS3Config.AuthenticationRegion = (string)_params["region"];
amazonS3Config.RegionEndpoint = regionEndpoint;
if (_params.ContainsKey("proxy"))
{
amazonS3Config.ProxyHost = (string)_params["proxy"];
amazonS3Config.ProxyPort = (int)_params["proxy_port"];
if (_params.ContainsKey("proxy_auth") && ((bool)_params["proxy_auth"]) == true)
{
var credential = new NetworkCredential();
credential.UserName = (string)_params["proxy_username"];
credential.Password = (string)_params["proxy_password"];
amazonS3Config.ProxyCredentials = credential;
}
}
// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/create-bucket-get-location-example.html#create-bucket-get-location-dotnet
amazonS3Config.ForcePathStyle =
_params.ContainsKey("use_path_style") ? (bool)_params["use_path_style"] : false;
this.connection = new AmazonS3Client(credentials, amazonS3Config);
// if (!this.connection.isBucketDnsCompatible(this.bucket)) {
// var logger = OC.server.getLogger();
// logger.debug("Bucket "' . this.bucket . '" This bucket name is not dns compatible, it may contain invalid characters.',
// ["app" => 'objectstore']);
// }
if (!AmazonS3Util.DoesS3BucketExistV2Async(this.connection, this.bucket).Result)
{
logger = OC.server.getLogger();
try {
logger.info("Bucket \"" + this.bucket + "\" does not exist - creating it.", ['app' => 'objectstore']);
// if (!this.connection.isBucketDnsCompatible(this.bucket)) {
// throw new Exception("The bucket will not be created because the name is not dns compatible, please correct it: " + this.bucket);
// }
var putBucketRequest = new PutBucketRequest
{
BucketName = this.bucket,
UseClientRegion = true
};
PutBucketResponse putBucketResponse = this.connection.PutBucketAsync(putBucketRequest).Result;
this.testTimeout();
} catch (Exception e) {
logger.logException(e, [
'message' => 'Invalid remote storage.',
'level' => ILogger::DEBUG,
'app' => 'objectstore',
]);
throw new Exception("Creation of bucket \"" + this.bucket + "\" failed. " + e.Message);
}
}
// google cloud's s3 compatibility doesn't like the EncodingType parameter
if (base_url.IndexOf("storage.googleapis.com") != -1)
{
// this.connection.getHandlerList().remove('s3.auto_encode');
}
return(this.connection);
}
public static IRequest RunMockRequest(AmazonWebServiceRequest request, IMarshaller <IRequest, AmazonWebServiceRequest> marshaller, AmazonS3Config config)
{
var pipeline = new RuntimePipeline(new List <IPipelineHandler>
{
new NoopPipelineHandler(),
new Signer(),
new EndpointResolver(),
new AmazonS3PostMarshallHandler(),
new Marshaller(),
new AmazonS3PreMarshallHandler(),
});
var requestContext = new RequestContext(config.LogMetrics, new Amazon.Runtime.Internal.Auth.S3Signer())
{
ClientConfig = config,
Marshaller = marshaller,
OriginalRequest = request,
Unmarshaller = null,
IsAsync = false,
ImmutableCredentials = new ImmutableCredentials("access key", "secret", "token")
};
var executionContext = new ExecutionContext(
requestContext,
new ResponseContext()
);
pipeline.InvokeSync(executionContext);
return(requestContext.Request);
}
private IRequest RunMockRequest(AmazonWebServiceRequest request, IMarshaller <IRequest, AmazonWebServiceRequest> marshaller, AmazonS3Config config)
{
var pipeline = new RuntimePipeline(new List <IPipelineHandler>
{
new S3ArnTestUtils.NoopPipelineHandler(),
new Marshaller()
});
var requestContext = new RequestContext(config.LogMetrics, new AWS4Signer())
{
ClientConfig = config,
Marshaller = marshaller,
OriginalRequest = request,
Unmarshaller = null,
IsAsync = false
};
var executionContext = new ExecutionContext(
requestContext,
new ResponseContext()
);
pipeline.InvokeSync(executionContext);
return(requestContext.Request);
}
public override async Task ActivatingAsync()
{
// Only create container if options are valid.
if (_shellSettings.State != Environment.Shell.Models.TenantState.Uninitialized &&
!string.IsNullOrEmpty(_options.S3HostEndpoint) &&
!string.IsNullOrEmpty(_options.S3BucketName) &&
!string.IsNullOrEmpty(_options.S3AccessKey) &&
!string.IsNullOrEmpty(_options.S3SecretKey) &&
_options.CreateContainer
)
{
_logger.LogDebug("Testing S3 Bucket {BucketName} existence", _options.S3BucketName);
var clientConfig = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(_options.S3Region),
ServiceURL = _options.S3HostEndpoint,
ForcePathStyle = true,
UseHttp = true,
};
var s3Client = new AmazonS3Client(_options.S3AccessKey, _options.S3SecretKey, clientConfig);
try
{
if (!await AmazonS3Util.DoesS3BucketExistV2Async(s3Client, _options.S3BucketName))
{
await s3Client.PutBucketAsync(new PutBucketRequest
{
BucketName = _options.S3BucketName,
BucketRegionName = _options.S3Region
});
var bucketLocation = await s3Client.GetBucketLocationAsync(new GetBucketLocationRequest
{
BucketName = _options.S3BucketName
});
_logger.LogDebug("S3 CreateBucket Request {ContainerName} send.", _options.S3BucketName);
if (bucketLocation.HttpStatusCode != HttpStatusCode.OK)
{
_logger.LogCritical(
"S3 Bucket not found {ContainerName}, after CreateBucket request was sent.",
_options.S3BucketName);
}
if (bucketLocation.Location != _options.S3Region)
{
_logger.LogCritical(
"S3 Bucket not found in specific Region {Region}, after CreateBucket request was sent.",
_options.S3Region);
}
var newPolicy = @"{
""Version"": ""2012-10-17"",
""Statement"":[{
""Action"":[""s3:GetObject""],
""Effect"":""Allow"",
""Principal"": {""AWS"": [""*""]},
""Resource"":[""arn:aws:s3:::" + _options.S3BucketName + @"/*""],
""Sid"":""""
}]}";
await s3Client.PutBucketPolicyAsync(new PutBucketPolicyRequest
{
BucketName = _options.S3BucketName,
Policy = newPolicy
});
var existingPolicy = await s3Client.GetBucketPolicyAsync(_options.S3BucketName);
if (existingPolicy.Policy != newPolicy)
{
_logger.LogCritical(
"S3 Bucket has this policy {ExistingPolicy}, instead of the expected Policy : {ExpectedPolicy}.",
newPolicy, existingPolicy.Policy);
}
}
_logger.LogDebug("S3 Bucket {ContainerName} created.", _options.S3BucketName);
}
catch (Exception e)
{
_logger.LogError(e, "Unable to create Azure Media Storage Container.");
}
}
}
public void SetQueueConfigurationTests()
{
var filterRule = new FilterRule("Prefix", "test/");
var s3Config = new AmazonS3Config();
using (var s3Client = new AmazonS3Client(s3Config))
using (var sqsClient = new AmazonSQSClient())
{
var createResponse = sqsClient.CreateQueue("events-test-" + DateTime.Now.Ticks);
var bucketName = S3TestUtils.CreateBucket(s3Client);
try
{
var queueArn = sqsClient.AuthorizeS3ToSendMessage(createResponse.QueueUrl, bucketName);
PutBucketNotificationRequest putRequest = new PutBucketNotificationRequest
{
BucketName = bucketName,
QueueConfigurations = new List <QueueConfiguration>
{
new QueueConfiguration
{
Id = "the-queue-test",
Queue = queueArn,
Events = { EventType.ObjectCreatedPut },
Filter = new Filter
{
S3KeyFilter = new S3KeyFilter
{
FilterRules = new List <FilterRule>
{
filterRule
}
}
}
}
}
};
s3Client.PutBucketNotification(putRequest);
var getResponse = s3Client.GetBucketNotification(bucketName);
Assert.AreEqual(1, getResponse.QueueConfigurations.Count);
Assert.AreEqual(1, getResponse.QueueConfigurations[0].Events.Count);
Assert.AreEqual(EventType.ObjectCreatedPut, getResponse.QueueConfigurations[0].Events[0]);
Assert.IsNotNull(getResponse.QueueConfigurations[0].Filter);
Assert.IsNotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter);
Assert.IsNotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules);
Assert.AreEqual(1, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules.Count);
Assert.AreEqual(filterRule.Name, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Name);
Assert.AreEqual(filterRule.Value, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Value);
Assert.AreEqual("the-queue-test", getResponse.QueueConfigurations[0].Id);
Assert.AreEqual(queueArn, getResponse.QueueConfigurations[0].Queue);
// Purge queue to remove test message sent configuration was setup.
sqsClient.PurgeQueue(createResponse.QueueUrl);
Thread.Sleep(TimeSpan.FromSeconds(1));
var putObjectRequest = new PutObjectRequest
{
BucketName = bucketName,
Key = "test/data.txt",
ContentBody = "Important Data"
};
s3Client.PutObject(putObjectRequest);
string messageBody = null;
for (int i = 0; i < 5 && messageBody == null; i++)
{
var receiveResponse = sqsClient.ReceiveMessage(new ReceiveMessageRequest {
QueueUrl = createResponse.QueueUrl, WaitTimeSeconds = 20
});
if (receiveResponse.Messages.Count != 0)
{
messageBody = receiveResponse.Messages[0].Body;
}
}
var evnt = S3EventNotification.ParseJson(messageBody);
Assert.AreEqual(1, evnt.Records.Count);
Assert.AreEqual(putObjectRequest.BucketName, evnt.Records[0].S3.Bucket.Name);
Assert.AreEqual(putObjectRequest.Key, evnt.Records[0].S3.Object.Key);
Assert.AreEqual(putObjectRequest.ContentBody.Length, evnt.Records[0].S3.Object.Size);
}
finally
{
sqsClient.DeleteQueue(createResponse.QueueUrl);
AmazonS3Util.DeleteS3BucketWithObjects(s3Client, bucketName);
}
}
}
public void SetQueueConfigurationTests()
{
var filterRule = new FilterRule("Prefix", "test/");
var s3Config = new AmazonS3Config();
using (var s3Client = new AmazonS3Client(s3Config))
using (var sqsClient = new AmazonSQSClient())
using (var stsClient = new AmazonSecurityTokenServiceClient())
{
var createResponse = sqsClient.CreateQueue("events-test-" + DateTime.Now.Ticks);
var bucketName = S3TestUtils.CreateBucketWithWait(s3Client);
try
{
var queueArn = sqsClient.AuthorizeS3ToSendMessage(createResponse.QueueUrl, bucketName);
PutBucketNotificationRequest putRequest = new PutBucketNotificationRequest
{
BucketName = bucketName,
QueueConfigurations = new List <QueueConfiguration>
{
new QueueConfiguration
{
Id = "the-queue-test",
Queue = queueArn,
Events = { EventType.ObjectCreatedPut },
Filter = new Filter
{
S3KeyFilter = new S3KeyFilter
{
FilterRules = new List <FilterRule>
{
filterRule
}
}
}
}
}
};
s3Client.PutBucketNotification(putRequest);
var getResponse = S3TestUtils.WaitForConsistency(() =>
{
var res = s3Client.GetBucketNotification(bucketName);
return(res.QueueConfigurations?.Count > 0 && res.QueueConfigurations[0].Id == "the-queue-test" ? res : null);
});
var getAttributeResponse = sqsClient.GetQueueAttributes(new GetQueueAttributesRequest
{
QueueUrl = createResponse.QueueUrl,
AttributeNames = new List <string> {
"All"
}
});
var policy = Policy.FromJson(getAttributeResponse.Policy);
var conditions = policy.Statements[0].Conditions;
Assert.AreEqual(2, conditions.Count);
var accountCondition = conditions.FirstOrDefault(x => string.Equals(x.ConditionKey, ConditionFactory.SOURCE_ACCOUNT_KEY));
Assert.IsNotNull(accountCondition);
Assert.AreEqual(ConditionFactory.StringComparisonType.StringEquals.ToString(), accountCondition.Type);
Assert.AreEqual(12, accountCondition.Values[0].Length);
var currentAccountId = stsClient.GetCallerIdentity(new GetCallerIdentityRequest()).Account;
Assert.AreEqual(currentAccountId, accountCondition.Values[0]);
Assert.AreEqual(1, getResponse.QueueConfigurations.Count);
Assert.AreEqual(1, getResponse.QueueConfigurations[0].Events.Count);
Assert.AreEqual(EventType.ObjectCreatedPut, getResponse.QueueConfigurations[0].Events[0]);
Assert.IsNotNull(getResponse.QueueConfigurations[0].Filter);
Assert.IsNotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter);
Assert.IsNotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules);
Assert.AreEqual(1, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules.Count);
Assert.AreEqual(filterRule.Name, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Name);
Assert.AreEqual(filterRule.Value, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Value);
Assert.AreEqual("the-queue-test", getResponse.QueueConfigurations[0].Id);
Assert.AreEqual(queueArn, getResponse.QueueConfigurations[0].Queue);
// Purge queue to remove test message sent configuration was setup.
sqsClient.PurgeQueue(createResponse.QueueUrl);
//We must wait 60 seconds or the next message being sent to the queue could be deleted while the queue is being purged.
Thread.Sleep(TimeSpan.FromSeconds(60));
var putObjectRequest = new PutObjectRequest
{
BucketName = bucketName,
Key = "test/data.txt",
ContentBody = "Important Data"
};
s3Client.PutObject(putObjectRequest);
string messageBody = null;
for (int i = 0; i < 5 && messageBody == null; i++)
{
var receiveResponse = sqsClient.ReceiveMessage(new ReceiveMessageRequest {
QueueUrl = createResponse.QueueUrl, WaitTimeSeconds = 20
});
if (receiveResponse.Messages.Count != 0)
{
messageBody = receiveResponse.Messages[0].Body;
}
}
var evnt = S3EventNotification.ParseJson(messageBody);
Assert.AreEqual(1, evnt.Records.Count);
Assert.AreEqual(putObjectRequest.BucketName, evnt.Records[0].S3.Bucket.Name);
Assert.AreEqual(putObjectRequest.Key, evnt.Records[0].S3.Object.Key);
Assert.AreEqual(putObjectRequest.ContentBody.Length, evnt.Records[0].S3.Object.Size);
Assert.IsNotNull(evnt.Records[0].S3.Object.Sequencer);
}
finally
{
sqsClient.DeleteQueue(createResponse.QueueUrl);
AmazonS3Util.DeleteS3BucketWithObjects(s3Client, bucketName);
}
}
}
private async void cmdUpload_Click(object sender, EventArgs e)
{
txtStatus.Text = string.Empty;
cmdUpload.Enabled = false;
USER_NAME = txtUserName.Text;
SECRET_KEY = txtSecretKey.Text;
BUCKET_NAME = txtBucketName.Text;
PICS_TO_UPLOAD = Convert.ToInt16(cmbPictureCount.Text);
PICS_UPLOADED = 0;
List <Picture> picturesToUpload = new List <Picture>();
DateTime startDate = DateTime.Now;
CancellationTokenSource cts = new CancellationTokenSource();
// Uri
string host = string.Empty;
string scheme = string.Empty;
string port = string.Empty;
bool useHttp = true;
if (USER_NAME.Length == 0 || SECRET_KEY.Length == 0 || BUCKET_NAME.Length == 0 || PICS_TO_UPLOAD == 0 || txtIpAddress.Text.Length == 0)
{
updateStatus("Please provide an endpoint ip, user name, secret key, bucket name and number of pictures to upload.");
cmdUpload.Enabled = true;
return;
}
try {
port = cmbPort.Text;
host = txtIpAddress.Text;
if (port == "9020")
{
scheme = "http://";
}
else
{
scheme = "https://";
useHttp = true;
}
ENDPOINT_URL = string.Format("{0}{1}:{2}", scheme, host, port);
//bool isValid = System.Net.IPAddress.TryParse(txtIpAddress.Text, out ip);
//if (isValid)
//{
// temp = string.Format("{0}{1}:{2}", scheme, ip, port);
//}
//else
//{
// updateStatus("Please provide a valid IP address.");
// cmdUpload.Enabled = true;
// return;
//}
//ENDPOINT_URL = new Uri(temp);
}
catch (Exception ex)
{
updateStatus(ex.Message);
cmdUpload.Enabled = true;
return;
}
var progress = new Progress <UploadProgress>(update =>
{
if (update.cancelLoop)
{
cts.Cancel();
}
if (update.errorMessage != null)
{
updateStatus(update.errorMessage);
}
else
{
double percentage = (double)PICS_UPLOADED / PICS_TO_UPLOAD;
updateStatus(string.Format("{0} of {1} uploaded - size {2}b - progress: {3}", Convert.ToString(PICS_UPLOADED), Convert.ToString(PICS_TO_UPLOAD), update.imageSize, percentage.ToString("0%")));
}
});
updateStatus("Checking that specified bucket exists.");
bool result = false;
await Task.Run(() =>
{
IProgress <UploadProgress> report = progress;
try
{
BasicAWSCredentials creds = new BasicAWSCredentials(USER_NAME, SECRET_KEY);
AmazonS3Config cc = new AmazonS3Config()
{
ServiceURL = ENDPOINT_URL,
ForcePathStyle = true,
UseHttp = useHttp,
Timeout = TimeSpan.FromSeconds(5),
MaxErrorRetry = 0
};
AmazonS3Client client = new AmazonS3Client(creds, cc);
ListBucketsRequest listBucketsRequest = new ListBucketsRequest();
ListBucketsResponse listBucketsResponse = client.ListBuckets(listBucketsRequest);
foreach (var bucket in listBucketsResponse.Buckets)
{
if (bucket.BucketName == BUCKET_NAME)
{
result = true;
}
}
if (!result)
{
report.Report(new UploadProgress {
errorMessage = "Unable to locate specified bucket. Did you provide the correct bucket name?"
});
}
}
catch (AmazonS3Exception amazonS3Exception)
{
report.Report(new UploadProgress {
errorMessage = string.Format("An error occured while communicating with ECS. Error: {0}", amazonS3Exception.Message)
});
}
catch (Exception ex)
{
report.Report(new UploadProgress {
errorMessage = "An error occured while attempting to access specified bucket. Please check all provided parameters."
});
}
});
if (!result)
{
cmdUpload.Enabled = true;
return;
}
updateStatus(string.Format("Successfully verified that bucket '{0}' exists.", BUCKET_NAME));
var engine = new FileHelperEngine <Pictures>();
var records = engine.ReadFile(ConfigurationManager.AppSettings["CSV_FILE_LOCATION"]);
int recordsIndex = 0;
foreach (var record in records)
{
recordsIndex++;
int height = record.height;
int width = record.width;
double longitude = record.longitude;
double latitude = record.latitude;
int viewCount = record.viewCount;
string fileName = record.fileName;
string filePath = string.Format("{0}\\{1}", ConfigurationManager.AppSettings["IMAGE_FOLDER_LOCATION"], fileName);
string thumbPath = string.Format("{0}\\thumbnails\\{1}", ConfigurationManager.AppSettings["IMAGE_FOLDER_LOCATION"], fileName);
picturesToUpload.Add(new Picture(recordsIndex, height, width, fileName, filePath, thumbPath, longitude, latitude, viewCount, this));
if (!(recordsIndex < PICS_TO_UPLOAD))
{
break;
}
}
await Task.Run(() => {
ParallelOptions po = new ParallelOptions();
po.CancellationToken = cts.Token;
try
{
Parallel.ForEach(picturesToUpload, po, (image) =>
{
try
{
image.uploadTheImage(progress);
}
finally
{
Interlocked.Increment(ref PICS_UPLOADED);
}
});
} catch (OperationCanceledException ex)
{
}
});
if (!cts.IsCancellationRequested)
{
DateTime endDate = DateTime.Now;
updateStatus(string.Format("Done: Process took: {0} seconds.", (endDate - startDate).TotalSeconds.ToString()));
}
cmdUpload.Enabled = true;
}
public MinioClient(AmazonS3Config clientConfig) : base(clientConfig)
{
}
