public S3BlobUploader(S3Settings s3Settings)
{
if (s3Settings == null)
throw new ArgumentNullException(nameof(s3Settings));
_s3Settings = s3Settings;
}
public RavenAwsS3Client(S3Settings s3Settings, Progress progress = null, Logger logger = null, CancellationToken?cancellationToken = null)
: base(s3Settings, progress, cancellationToken)
{
if (string.IsNullOrWhiteSpace(s3Settings.BucketName))
{
throw new ArgumentException("AWS Bucket name cannot be null or empty");
}
if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey))
{
throw new ArgumentException("AWS Access Key cannot be null or empty");
}
if (string.IsNullOrWhiteSpace(s3Settings.CustomServerUrl))
{
if (string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
{
throw new ArgumentException("AWS region name cannot be null or empty");
}
}
else
{
_customS3ServerUrl = new Uri(s3Settings.CustomServerUrl);
}
AwsRegion = s3Settings.AwsRegionName == null ? string.Empty : s3Settings.AwsRegionName.ToLower();
_bucketName = s3Settings.BucketName;
_logger = logger;
}
public bool Equals(S3Settings other)
{
if (other == null)
{
return(false);
}
if (WasEnabled(other))
{
return(true);
}
if (other.AwsRegionName != AwsRegionName)
{
return(false);
}
if (other.BucketName != BucketName)
{
return(false);
}
if (other.RemoteFolderName != RemoteFolderName)
{
return(false);
}
return(true);
}
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly1()
{
var s3Settings = new S3Settings
{
AwsAccessKey = "AKIAIOSFODNN7EXAMPLE",
AwsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
AwsRegionName = "us-east-1",
BucketName = "examplebucket"
};
using (var client = new RavenAwsS3Client(s3Settings))
{
var date = new DateTime(2013, 5, 24);
var stream = new MemoryStream(Encoding.UTF8.GetBytes("Welcome to Amazon S3."));
var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream);
Assert.Equal("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", payloadHash);
var url = client.GetUrl() + "/test%24file.text";
var headers = new Dictionary <string, string>
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(date) },
{ "x-amz-content-sha256", payloadHash },
{ "x-amz-storage-class", "REDUCED_REDUNDANCY" },
{ "Date", date.ToString("R") },
{ "Host", "s3.amazonaws.com" }
};
var auth = client.CalculateAuthorizationHeaderValue(HttpMethods.Put, url, date, headers);
Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme);
Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=d2dd2e48b10d2cb89c271a6464d0748686c158b5fde44e8d83936fd9b30b5c4c", auth.Parameter);
}
}
public void AuthorizationHeaderValueForAwsS3ShouldBeCalculatedCorrectly2()
{
var s3Settings = new S3Settings
{
AwsAccessKey = "AKIAIOSFODNN7EXAMPLE",
AwsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
AwsRegionName = "us-east-1",
BucketName = "examplebucket"
};
using (var client = new RavenAwsS3Client(s3Settings))
{
var date = new DateTime(2013, 5, 24);
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
Assert.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", payloadHash);
var url = client.GetUrl() + "/test.txt";
var headers = new Dictionary <string, string>
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(date) },
{ "x-amz-content-sha256", payloadHash },
{ "Date", date.ToString("R") },
{ "Host", "s3.amazonaws.com" },
{ "Range", "bytes=0-9" }
};
var auth = client.CalculateAuthorizationHeaderValue(HttpMethods.Get, url, date, headers);
Assert.Equal("AWS4-HMAC-SHA256", auth.Scheme);
Assert.Equal("Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=819484c483cfb97d16522b1ac156f87e61677cc8f1f2545c799650ef178f4aa8", auth.Parameter);
}
}
/// <summary>
/// Gets relevant app setting from appSettings.json and maps to POCO classes and configures singleton for DI
/// </summary>
/// <param name="services"></param>
/// <param name="configuration"></param>
public static void ConfigureS3Settings(this IServiceCollection services, IConfiguration configuration)
{
var s3Settings = new S3Settings();
configuration.Bind("S3Settings", s3Settings);
services.AddSingleton <S3Settings>(s3Settings); // add singleton for DI
}
public CustomProfile(S3Settings s3Settings)
{
CreateMap <Category, CategoryDto>()
.ForMember(d => d.Description, opt => opt.MapFrom(e => e.Description))
.ForMember(d => d.Id, opt => opt.MapFrom(e => e.CategoryId))
.ForMember(d => d.ImageUrl, opt => opt.MapFrom(e => $"{s3Settings.ImgBaseUrl}{e.CategoryId}/{e.ImageName}"));
}
public S3Settings GetS3Settings(string subPath = null, [CallerMemberName] string caller = null)
{
var s3Settings = _isCustom ? CustomS3FactAttribute.S3Settings : AmazonS3FactAttribute.S3Settings;
if (s3Settings == null)
{
return(null);
}
var remoteFolderName = $"{s3Settings.RemoteFolderName}/{_remoteFolderName}/{GetRemoteFolder(caller)}";
if (string.IsNullOrEmpty(subPath) == false)
{
remoteFolderName = $"{remoteFolderName}/{subPath}";
}
var settings = new S3Settings
{
BucketName = s3Settings.BucketName,
RemoteFolderName = remoteFolderName,
AwsAccessKey = s3Settings.AwsAccessKey,
AwsSecretKey = s3Settings.AwsSecretKey,
AwsRegionName = s3Settings.AwsRegionName
};
if (_isCustom)
{
settings.CustomServerUrl = s3Settings.CustomServerUrl;
settings.ForcePathStyle = s3Settings.ForcePathStyle;
}
return(settings);
}
public DocumentReadAnalyzeService(ITextractTextAnalysisService analysisService, ITextractTextDetectionService detectionService, S3Settings s3Settings, ILogger logger)
{
_analysisService = analysisService;
_detectionService = detectionService;
_s3Settings = s3Settings;
_logger = logger;
}
public static void UpdateSettingsForS3(S3Settings s3Settings, string databaseName)
{
if (s3Settings != null)
{
s3Settings.RemoteFolderName = GetUpdatedPath(s3Settings.RemoteFolderName, databaseName);
}
}
public static IServiceCollection AddInfrastructure(this IServiceCollection services,
IConfiguration configuration)
{
services.AddDbContext <ExpenseContext>(options =>
options.UseNpgsql(configuration.GetConnectionString("ExpenseDbConnection"),
b => b.MigrationsAssembly(typeof(ExpenseContext).Assembly.FullName)));
services.AddScoped <IExpenseContext>(provider => provider.GetService <ExpenseContext>());
services.AddDefaultIdentity <ApplicationUser>()
.AddRoles <IdentityRole>()
.AddEntityFrameworkStores <ExpenseContext>();
var s3Settings = new S3Settings();
configuration.Bind(nameof(s3Settings), s3Settings);
services.AddSingleton(s3Settings);
var mapperCfg = new MapperConfiguration(cfg =>
{
cfg.AddProfile(new CustomProfile(s3Settings));
});
var mapperCustomProfile = mapperCfg.CreateMapper();
services.AddSingleton(mapperCustomProfile);
return(services);
}
private async Task UploadToS3(
S3Settings settings,
Stream stream,
string folderName,
string fileName,
Progress progress,
string archiveDescription)
{
using (var client = new RavenAwsS3Client(settings.AwsAccessKey, settings.AwsSecretKey,
settings.AwsRegionName, settings.BucketName, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName, folderName, fileName);
await client.PutObject(key, stream, new Dictionary <string, string>
{
{ "Description", archiveDescription }
});
if (_logger.IsInfoEnabled)
{
_logger.Info(string.Format($"Successfully uploaded backup file '{fileName}' " +
$"to S3 bucket named: {settings.BucketName}, " +
$"with key: {key}"));
}
}
}
private IAmazonS3 GetS3Config(S3Settings s3settings)
{
var region = RegionEndpoint.GetBySystemName(s3settings.AWSRegion);
return(new AmazonS3Client(
new BasicAWSCredentials(s3settings.AWSAccessKey,
s3settings.AWSSecretKey), region));
}
public StoriesService(IAmazonS3Repository amazonS3Repository, IMapper mapper, IStoriesRepository storiesRepository, IFeedService feedService, IOptions <S3Settings> s3Settings)
{
_amazonS3Repository = amazonS3Repository;
_mapper = mapper;
_storiesRepository = storiesRepository;
_feedService = feedService;
_s3Settings = s3Settings.Value;
}
public UsersService(IUsersRepository usersRepository, ICryptographyService cryptographyService, IMapper mapper, IAmazonS3Repository amazonS3Repository, IFeedService feedService, IOptions <S3Settings> s3Settings)
{
_usersRepository = usersRepository;
_cryptographyService = cryptographyService;
_mapper = mapper;
_amazonS3Repository = amazonS3Repository;
_feedService = feedService;
_s3Settings = s3Settings.Value;
}
public RavenAwsS3Client(S3Settings s3Settings, Progress progress = null, CancellationToken?cancellationToken = null)
: base(s3Settings, progress, cancellationToken)
{
if (string.IsNullOrWhiteSpace(s3Settings.BucketName))
{
throw new ArgumentException("AWS Bucket name cannot be null or empty");
}
_bucketName = s3Settings.BucketName;
}
public PhotosService(IPhotosRepository photosRepository, IAmazonS3Repository amazonS3Repository, IHashtagsService hashtagsService, IMapper mapper, IFeedService feedService, IOptions <S3Settings> s3Settings)
{
_photosRepository = photosRepository;
_amazonS3Repository = amazonS3Repository;
_hashtagsService = hashtagsService;
_mapper = mapper;
_feedService = feedService;
_s3Settings = s3Settings.Value;
}
public ControllerTestBase()
{
_S3Settings = new S3Settings()
{
ImgBaseUrl = "http://baseImageurl.com/",
ImageBucket = "ExpensesTestBucket"
};
_mediatorFake = A.Fake <IMediator>();
_loggerFake = A.Fake <ILogger <T> >();
ConfigureMediatorFake(_mediatorFake);
}
public override DynamicJsonValue ToJson()
{
var json = base.ToJson();
json[nameof(LocalSettings)] = LocalSettings?.ToJson();
json[nameof(S3Settings)] = S3Settings?.ToJson();
json[nameof(AzureSettings)] = AzureSettings?.ToJson();
json[nameof(GlacierSettings)] = GlacierSettings?.ToJson();
json[nameof(GoogleCloudSettings)] = GoogleCloudSettings?.ToJson();
json[nameof(FtpSettings)] = FtpSettings?.ToJson();
return(json);
}
public S3StorageService(string name, IOptions <S3Settings> s3Settings, bool allowDirectDownload, bool allowDirectUpload, ILogger <StorageServiceProvider> logger)
{
Type = StorageServiceProvider.AWS_S3;
Name = name;
S3Settings = s3Settings.Value;
SupportsDirectDownload = true;
AllowDirectDownload = allowDirectDownload;
SupportsDirectUpload = true;
AllowDirectUpload = allowDirectUpload;
_logger = logger;
_region = RegionEndpoint.GetBySystemName(S3Settings.RegionSystemName);
}
private void ConfigureServices()
{
_storageServices = new Dictionary <string, IStorageService>();
foreach (var ss in _hiarcSettings.StorageServices)
{
if (ss.Provider == AWS_S3)
{
var settings = new S3Settings
{
AccessKeyId = ((dynamic)ss.Config).AccessKeyId,
SecretAccessKey = ((dynamic)ss.Config).SecretAccessKey,
RegionSystemName = ((dynamic)ss.Config).RegionSystemName,
Bucket = ((dynamic)ss.Config).Bucket
};
IOptions <S3Settings> s3Settings = Options.Create(settings);
IStorageService s3Service = new S3StorageService(ss.Name, s3Settings, ss.AllowDirectDownload, ss.AllowDirectUpload, _logger);
_storageServices.Add(ss.Name, s3Service);
}
else if (ss.Provider == AZURE_BLOB_STORAGE)
{
var settings = new AzureSettings
{
StorageConnectionString = ((dynamic)ss.Config).StorageConnectionString,
Container = ((dynamic)ss.Config).Container
};
IOptions <AzureSettings> azureSettings = Options.Create(settings);
IStorageService azureService = new AzureStorageService(ss.Name, azureSettings, ss.AllowDirectDownload, ss.AllowDirectUpload, _logger);
_storageServices.Add(ss.Name, azureService);
}
else if (ss.Provider == GOOGLE_STORAGE)
{
var settings = new GoogleSettings
{
ServiceAccountCredential = ((dynamic)ss.Config).ServiceAccountCredential,
Bucket = ((dynamic)ss.Config).Bucket
};
IOptions <GoogleSettings> googleSettings = Options.Create(settings);
IStorageService googleService = new GoogleStorageService(ss.Name, googleSettings, ss.AllowDirectDownload, ss.AllowDirectUpload, _logger);
_storageServices.Add(ss.Name, googleService);
}
else
{
throw new Exception($"Unsupported storage service provider: {ss.Provider}");
}
}
}
static S3Fact()
{
var s3SettingsString = Environment.GetEnvironmentVariable(S3CredentialEnvironmentVariable, EnvironmentVariableTarget.User);
try
{
S3Settings = JsonConvert.DeserializeObject <S3Settings>(s3SettingsString);
}
catch (Exception e)
{
ParsingError = e.ToString();
}
}
private void DeleteFromS3(S3Settings settings)
{
using (var client = new RavenAwsS3Client(settings, _settings.Configuration, progress: null, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.DeleteObject(key);
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportDeletion(S3Name)} bucket named: {settings.BucketName}, with key: {key}");
}
}
}
static AmazonS3FactAttribute()
{
var s3SettingsString = Environment.GetEnvironmentVariable(S3CredentialEnvironmentVariable);
try
{
S3Settings = JsonConvert.DeserializeObject <S3Settings>(s3SettingsString);
}
catch (Exception e)
{
ParsingError = e.ToString();
}
}
public FileUploadService()
{
// hard-coded config for POC
S3Settings tempSetting = new S3Settings
{
OriginalBucketName = "praba.bucket",
OriginalBucketUrl = "https://s3-ap-southeast-2.amazonaws.com/praba.bucket/",
AWSRegion = "ap-southeast-2",
AWSAccessKey = "AKIAJAXHRPK5IGGDAMJA",
AWSSecretKey = "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"
};
S3Clinet = GetS3Config(tempSetting);
}
public S3NoteStorageService(IOptions <S3Settings> s3Settings, IDistributedCache cache, DistributedCacheEntryOptions cacheEntryOptions, ILoggerFactory loggerFactory)
{
_cache = cache;
_cacheEntryOptions = cacheEntryOptions;
_logger = loggerFactory.CreateLogger <S3NoteStorageService>();
_s3Settings = s3Settings.Value;
var region = RegionEndpoint.GetBySystemName(s3Settings.Value.AWSRegion);
_s3Client = new AmazonS3Client(
new BasicAWSCredentials(
s3Settings.Value.AWSAccessKey,
s3Settings.Value.AWSSecretKey),
region);
}
protected override void ValidateImpl(ref List <string> errors)
{
if (S3Settings != null)
{
if (S3Settings.HasSettings() == false)
{
errors.Add($"{nameof(S3Settings)} has no valid setting. '{nameof(S3Settings.BucketName)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (AzureSettings != null)
{
if (AzureSettings.HasSettings() == false)
{
errors.Add($"{nameof(AzureSettings)} has no valid setting. '{nameof(AzureSettings.StorageContainer)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (GlacierSettings != null)
{
if (GlacierSettings.HasSettings() == false)
{
errors.Add($"{nameof(GlacierSettings)} has no valid setting. '{nameof(GlacierSettings.VaultName)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (GoogleCloudSettings != null)
{
if (GoogleCloudSettings.HasSettings() == false)
{
errors.Add($"{nameof(GoogleCloudSettings)} has no valid setting. '{nameof(GoogleCloudSettings.BucketName)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (FtpSettings != null)
{
if (FtpSettings.HasSettings() == false)
{
errors.Add($"{nameof(FtpSettings)} has no valid setting. '{nameof(FtpSettings.Port)}' is 0 and '{nameof(FtpSettings.Url)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (LocalSettings != null)
{
if (LocalSettings.HasSettings() == false)
{
errors.Add($"{nameof(LocalSettings)} has no valid setting. '{nameof(LocalSettings.FolderPath)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
}
static CustomS3FactAttribute()
{
var strSettings = Environment.GetEnvironmentVariable(S3CredentialEnvironmentVariable);
if (string.IsNullOrEmpty(strSettings))
{
return;
}
try
{
S3Settings = JsonConvert.DeserializeObject <S3Settings>(strSettings);
}
catch (Exception e)
{
ParsingError = e.ToString();
}
}
private void UploadToS3(S3Settings settings, Stream stream, Progress progress)
{
using (var client = new RavenAwsS3Client(settings, _settings.Configuration, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.PutObject(key, stream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription() }
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportSuccess(S3Name)} bucket named: {settings.BucketName}, with key: {key}");
}
var runner = new S3RetentionPolicyRunner(_retentionPolicyParameters, client);
runner.Execute();
}
}
static AmazonS3TheoryAttribute()
{
var s3SettingsString = Environment.GetEnvironmentVariable(S3CredentialEnvironmentVariable);
if (s3SettingsString == null)
{
EnvVariableMissing = true;
return;
}
try
{
_s3Settings = JsonConvert.DeserializeObject <S3Settings>(s3SettingsString);
}
catch (Exception e)
{
ParsingError = e.ToString();
}
}
static CustomS3FactAttribute()
{
var strSettings = Environment.GetEnvironmentVariable(S3CredentialEnvironmentVariable);
if (strSettings == null)
{
EnvVariableMissing = true;
return;
}
try
{
_s3Settings = JsonConvert.DeserializeObject <S3Settings>(strSettings);
}
catch (Exception e)
{
ParsingError = e.ToString();
}
}
public S3LinkCreator(S3Settings s3Settings)
{
_s3Settings = s3Settings;
_linkManager = new LinkManager();
}