/// <summary>Add SimpleS3 services to a service collection.</summary>
/// <param name="collection">The service collection</param>
public static IClientBuilder AddAmazonS3(this IServiceCollection collection)
{
ICoreBuilder coreBuilder = SimpleS3CoreServices.AddSimpleS3Core(collection);
coreBuilder.UseAmazonS3();
IHttpClientBuilder httpBuilder = coreBuilder.UseHttpClientFactory();
httpBuilder.UseDefaultHttpPolicy();
coreBuilder.Services.AddSingleton(x =>
{
//We have to call a specific constructor for dependency injection
IObjectClient objectClient = x.GetRequiredService <IObjectClient>();
IBucketClient bucketClient = x.GetRequiredService <IBucketClient>();
IMultipartClient multipartClient = x.GetRequiredService <IMultipartClient>();
IMultipartTransfer multipartTransfer = x.GetRequiredService <IMultipartTransfer>();
ITransfer transfer = x.GetRequiredService <ITransfer>();
ISignedObjectClient?signedObjectClient = x.GetRequiredService <ISignedObjectClient>();
return(new AmazonS3Client(objectClient, bucketClient, multipartClient, multipartTransfer, transfer, signedObjectClient));
});
//Add the client as the interface too
coreBuilder.Services.AddSingleton <ISimpleClient>(x => x.GetRequiredService <AmazonS3Client>());
return(new ClientBuilder(collection, httpBuilder, coreBuilder));
}
private void Initialize(IOptions <S3Config> options, INetworkDriver networkDriver, ILoggerFactory loggerFactory)
{
Assembly assembly = typeof(S3Config).Assembly;
SimpleServiceProvider provider = new SimpleServiceProvider(new Tuple <Type, object>(typeof(IOptions <S3Config>), options));
IEnumerable <IValidator> validators = CreateInstances <IValidator>(assembly, provider);
IEnumerable <IRequestMarshal> requestMarshals = CreateInstances <IRequestMarshal>(assembly, provider);
IEnumerable <IResponseMarshal> responseMarshals = CreateInstances <IResponseMarshal>(assembly, provider);
ValidatorFactory validatorFactory = new ValidatorFactory(validators);
IMarshalFactory marshalFactory = new MarshalFactory(requestMarshals, responseMarshals);
IScopeBuilder scopeBuilder = new ScopeBuilder(options);
ISigningKeyBuilder signingKeyBuilder = new SigningKeyBuilder(options, loggerFactory.CreateLogger <SigningKeyBuilder>());
ISignatureBuilder signatureBuilder = new SignatureBuilder(signingKeyBuilder, scopeBuilder, loggerFactory.CreateLogger <SignatureBuilder>(), options);
IAuthorizationBuilder authorizationBuilder = new AuthorizationHeaderBuilder(options, scopeBuilder, signatureBuilder, loggerFactory.CreateLogger <AuthorizationHeaderBuilder>());
DefaultRequestHandler requestHandler = new DefaultRequestHandler(options, validatorFactory, marshalFactory, networkDriver, authorizationBuilder, Enumerable.Empty <IRequestStreamWrapper>(), loggerFactory.CreateLogger <DefaultRequestHandler>());
ObjectOperations objectOperations = new ObjectOperations(requestHandler, Enumerable.Empty <IRequestWrapper>(), Enumerable.Empty <IResponseWrapper>());
_objectClient = new S3ObjectClient(objectOperations);
BucketOperations bucketOperations = new BucketOperations(requestHandler);
_bucketClient = new S3BucketClient(bucketOperations);
MultipartOperations multipartOperations = new MultipartOperations(requestHandler, Enumerable.Empty <IRequestWrapper>(), Enumerable.Empty <IResponseWrapper>());
_multipartClient = new S3MultipartClient(multipartOperations, objectOperations);
Transfer = new Transfer(objectOperations, multipartOperations);
}
public IntegrationTest(ITestOutputHelper output)
{
_output = output;
dynamic config = ConfigurationLoader.LoadConfiguration("secrets.json");
//AWS
dynamic aws = config.aws;
string awsId = aws.id;
string awsKey = aws.key;
string awsRegion = aws.region;
awsClient = BucketClientFactory.CreateClient(CloudServiceProvider.AWS, new AWSCredential(awsId, awsKey, awsRegion));
awsTargetBucket = aws.bucket;
//DO
dynamic DO = config.DO;
string doId = DO.id;
string doKey = DO.key;
string doRegion = DO.region;
doClient = BucketClientFactory.CreateClient(CloudServiceProvider.DigitalOcean, new DigitalOceanCredential(doId, doKey, doRegion));
doTargetBucket = DO.bucket;
//Azure
dynamic azure = config.azure;
string azureId = azure.id;
string azurekey = azure.key;
azureClient = BucketClientFactory.CreateClient(CloudServiceProvider.Azure, new AzureCredential(azureId, azurekey));
azureTargetBucket = azure.bucket;
}
internal AWSBucket(string key, AWSHttpClient client, string region, AWSBucketClient bucketClient)
{
_region = region;
_client = client;
_key = key;
_bucketClient = bucketClient;
}
public Factory(ICredential awsCred, ICredential doCred, ICredential azureCred, ICredential gcpCred)
{
AWSBucket = BucketClientFactory.CreateClient(CloudServiceProvider.AWS, awsCred);
DOBucket = BucketClientFactory.CreateClient(CloudServiceProvider.DigitalOcean, doCred);
AzureBucket = BucketClientFactory.CreateClient(CloudServiceProvider.Azure, azureCred);
GCPBucket = BucketClientFactory.CreateClient(CloudServiceProvider.GCP, gcpCred);
}
private static async Task Main(string[] args)
{
Console.WriteLine("You are about to set up the test environment for the provider tests");
S3Provider selectedProvider = UtilityHelper.SelectProvider();
Console.WriteLine();
string profileName = UtilityHelper.GetProfileName(selectedProvider);
using ServiceProvider provider = UtilityHelper.CreateSimpleS3(selectedProvider, profileName, true);
IProfile profile = UtilityHelper.GetOrSetupProfile(provider, selectedProvider, profileName);
string bucketName = UtilityHelper.GetTestBucket(profile);
IBucketClient bucketClient = provider.GetRequiredService <IBucketClient>();
Console.WriteLine($"Setting up bucket '{bucketName}'");
bool success = await TryCreateBucketAsync(bucketClient, bucketName);
if (success)
{
if (selectedProvider == S3Provider.AmazonS3)
{
await PostConfigureAmazonS3(bucketClient, bucketName).ConfigureAwait(false);
}
else if (selectedProvider == S3Provider.BackBlazeB2)
{
await PostConfigureBackBlazeB2(bucketClient, bucketName).ConfigureAwait(false);
}
}
}
public S3Client(IObjectClient objectClient, IBucketClient bucketClient, IMultipartClient multipartClient)
{
_objectClient = objectClient;
_bucketClient = bucketClient;
_multipartClient = multipartClient;
Transfer = new Transfer(_objectClient.ObjectOperations, _multipartClient.MultipartOperations);
}
protected void Initialize(IObjectClient objectClient, IBucketClient bucketClient, IMultipartClient multipartClient, IMultipartTransfer multipartTransfer, ITransfer transfer, ISignedObjectClient signedObjectClient)
{
_objectClient = objectClient;
_bucketClient = bucketClient;
_multipartClient = multipartClient;
_multipartTransfer = multipartTransfer;
_transfer = transfer;
_signedObjectClient = signedObjectClient;
}
private static async IAsyncEnumerable <S3Bucket> ListAllBucketsAsync(IBucketClient client)
{
ListBucketsResponse response = await client.ListBucketsAsync().ConfigureAwait(false);
foreach (S3Bucket bucket in response.Buckets)
{
yield return(bucket);
}
}
private void Build(IServiceCollection services)
{
_serviceProvider = services.BuildServiceProvider();
IObjectClient objectClient = _serviceProvider.GetRequiredService <IObjectClient>();
IBucketClient bucketClient = _serviceProvider.GetRequiredService <IBucketClient>();
IMultipartClient multipartClient = _serviceProvider.GetRequiredService <IMultipartClient>();
IMultipartTransfer multipartTransfer = _serviceProvider.GetRequiredService <IMultipartTransfer>();
ITransfer transfer = _serviceProvider.GetRequiredService <ITransfer>();
ISignedObjectClient signedObjectClient = _serviceProvider.GetRequiredService <ISignedObjectClient>();
Client = new SimpleClient(objectClient, bucketClient, multipartClient, multipartTransfer, transfer, signedObjectClient);
}
private static async Task PostConfigureBackBlazeB2(IBucketClient bucketClient, string bucketName)
{
Console.WriteLine("Adding lock configuration");
PutBucketLockConfigurationResponse putLockResp = await bucketClient.PutBucketLockConfigurationAsync(bucketName, true).ConfigureAwait(false);
if (putLockResp.IsSuccess)
{
Console.WriteLine("Successfully applied lock configuration.");
}
else
{
Console.Error.WriteLine("Failed to apply lock configuration.");
}
}
public static ICoreBuilder UseS3Client(this ICoreBuilder builder)
{
builder.Services.TryAddSingleton(x =>
{
//We have to call a specific constructor for dependency injection
IObjectClient objectClient = x.GetRequiredService <IObjectClient>();
IBucketClient bucketClient = x.GetRequiredService <IBucketClient>();
IMultipartClient multipartClient = x.GetRequiredService <IMultipartClient>();
return(new S3Client(objectClient, bucketClient, multipartClient));
});
//Add the client as the interface too
builder.Services.TryAddSingleton <IClient>(x => x.GetRequiredService <S3Client>());
return(builder);
}
private static async Task PostConfigureAmazonS3(IBucketClient bucketClient, string bucketName)
{
Console.WriteLine("Adding lock configuration");
PutBucketLockConfigurationResponse putLockResp = await bucketClient.PutBucketLockConfigurationAsync(bucketName, true).ConfigureAwait(false);
if (putLockResp.IsSuccess)
{
Console.WriteLine("Successfully applied lock configuration.");
}
else
{
Console.Error.WriteLine("Failed to apply lock configuration.");
}
List <S3Rule> rules = new List <S3Rule>
{
new S3Rule("ExpireAll", true)
{
AbortIncompleteMultipartUploadDays = 1,
NonCurrentVersionExpirationDays = 1,
Expiration = new S3Expiration(1),
Filter = new S3Filter {
Prefix = ""
}
}
};
Console.WriteLine("Adding lifecycle configuration");
PutBucketLifecycleConfigurationResponse putLifecycleResp = await bucketClient.PutBucketLifecycleConfigurationAsync(bucketName, rules).ConfigureAwait(false);
if (putLifecycleResp.IsSuccess)
{
Console.WriteLine("Successfully applied lifecycle configuration.");
}
else
{
Console.Error.WriteLine("Failed to apply lifecycle configuration.");
}
}
private static async Task <bool> TryCreateBucketAsync(IBucketClient bucketClient, string bucketName)
{
HeadBucketResponse headResp = await bucketClient.HeadBucketAsync(bucketName);
//Bucket already exist - we return true to apply post-configuration
if (headResp.StatusCode == 200)
{
return(true);
}
CreateBucketResponse createResp = await bucketClient.CreateBucketAsync(bucketName, r => r.EnableObjectLocking = true).ConfigureAwait(false);
if (createResp.IsSuccess)
{
Console.WriteLine($"Successfully created '{bucketName}'.");
return(true);
}
Console.Error.WriteLine($"Failed to create '{bucketName}'. Error: " + createResp.Error?.Message);
return(false);
}
/// <summary>List all buckets</summary>
public static async IAsyncEnumerable <S3Bucket> ListAllBucketsAsync(this IBucketClient client, Action <ListBucketsRequest>?config = null, [EnumeratorCancellation] CancellationToken token = default)
{
ListBucketsResponse response = await client.ListBucketsAsync(config, token).ConfigureAwait(false);
if (!response.IsSuccess)
{
throw new S3RequestException(response.StatusCode, "Request failed");
}
if (token.IsCancellationRequested)
{
yield break;
}
foreach (S3Bucket bucket in response.Buckets)
{
if (token.IsCancellationRequested)
{
yield break;
}
yield return(bucket);
}
}
private static async Task Main(string[] args)
{
IConfigurationRoot root = new ConfigurationBuilder()
.AddJsonFile("Config.json", false)
.Build();
ServiceCollection services = new ServiceCollection();
services.Configure <S3Config>(root);
IS3ClientBuilder clientBuilder = services.AddSimpleS3();
clientBuilder.CoreBuilder.UseProfileManager()
.BindConfigToDefaultProfile()
.UseDataProtection();
IConfigurationSection proxySection = root.GetSection("Proxy");
if (proxySection != null && proxySection["UseProxy"].Equals("true", StringComparison.OrdinalIgnoreCase))
{
clientBuilder.HttpBuilder.WithProxy(proxySection["ProxyAddress"]);
}
using (ServiceProvider provider = services.BuildServiceProvider())
{
IProfileManager manager = provider.GetRequiredService <IProfileManager>();
IProfile? profile = manager.GetDefaultProfile();
//If profile is null, then we do not yet have a profile stored on disk. We use ConsoleSetup as an easy and secure way of asking for credentials
if (profile == null)
{
Console.WriteLine("No profile found. Starting setup.");
ConsoleSetup.SetupDefaultProfile(manager);
}
IBucketClient bucketClient = provider.GetRequiredService <IBucketClient>();
string bucketName = root["BucketName"];
Console.WriteLine($"Setting up bucket '{bucketName}'");
HeadBucketResponse headResp = await bucketClient.HeadBucketAsync(bucketName).ConfigureAwait(false);
if (headResp.IsSuccess)
{
Console.WriteLine($"'{bucketName}' already exist.");
}
else
{
Console.WriteLine($"'{bucketName}' does not exist - creating.");
CreateBucketResponse createResp = await bucketClient.CreateBucketAsync(bucketName, x => x.EnableObjectLocking = true).ConfigureAwait(false);
if (createResp.IsSuccess)
{
Console.WriteLine($"Successfully created '{bucketName}'.");
}
else
{
Console.Error.WriteLine($"Failed to create '{bucketName}'. Exiting.");
return;
}
}
Console.WriteLine("Adding lock configuration");
PutBucketLockConfigurationResponse putLockResp = await bucketClient.PutBucketLockConfigurationAsync(bucketName, true).ConfigureAwait(false);
if (putLockResp.IsSuccess)
{
Console.WriteLine("Successfully applied lock configuration.");
}
else
{
Console.Error.WriteLine("Failed to apply lock configuration.");
}
List <S3Rule> rules = new List <S3Rule>
{
new S3Rule("ExpireAll", true)
{
AbortIncompleteMultipartUploadDays = 1,
NonCurrentVersionExpirationDays = 1,
Expiration = new S3Expiration(1),
Filter = new S3Filter {
Prefix = ""
}
}
};
Console.WriteLine("Adding lifecycle configuration");
PutBucketLifecycleConfigurationResponse putLifecycleResp = await bucketClient.PutBucketLifecycleConfigurationAsync(bucketName, rules).ConfigureAwait(false);
if (putLifecycleResp.IsSuccess)
{
Console.WriteLine("Successfully applied lifecycle configuration.");
}
else
{
Console.Error.WriteLine("Failed to apply lifecycle configuration.");
}
}
}
public SimpleClient(IObjectClient objectClient, IBucketClient bucketClient, IMultipartClient multipartClient, IMultipartTransfer multipartTransfer, ITransfer transfer, ISignedObjectClient signedObject)
{
Initialize(objectClient, bucketClient, multipartClient, multipartTransfer, transfer, signedObject);
}
private static async Task AsyncMain(string[] arg, Factory fac)
{
var argument = Console.ReadLine();
var argo = argument.Split(' ').Where(x => x.Trim() != "").ToArray();
var clientType = argo[0];
var args = argo.Skip(1).ToArray();
try
{
IBucketClient client = fac.GetBucket(clientType.ToEnum());
switch (args[0].ToLower())
{
case "bucket":
switch (args[1].ToLower())
{
case "exist":
var exist = await client.ExistBucket(args[2]);
Console.WriteLine(args[2] + " " + (exist ? "exists!" : "does not exist"));
break;
case "create":
var createResp = await client.CreateBucket(args[2]);
Console.WriteLine(JsonConvert.SerializeObject(createResp));
break;
case "del": goto case "delete";
case "delete":
var delResp = await client.DeleteBucket(args[2]);
Console.WriteLine(JsonConvert.SerializeObject(delResp));
break;
case "policy":
var policyResp = await client.SetReadPolicy(args[2], args[3] == "public"?ReadAccess.Public : ReadAccess.Private);
Console.WriteLine(JsonConvert.SerializeObject(policyResp));
break;
case "cors":
var corsResp = await client.SetGETCors(args[2], args.Length > 3?args.Skip(3).ToArray() : new string[] { });
Console.Write(JsonConvert.SerializeObject(corsResp));
break;
default:
Console.WriteLine("Unknown command!");
break;
}
break;
case "blob":
var bucketKey = args[2];
var bucket = await client.GetBucket(bucketKey);
var argv = args[3];
switch (args[1].ToLower())
{
case "exist":
var exist = await bucket.ExistBlob(argv);
Console.WriteLine(argv + " " + (exist ? "exists!" : "does not exist"));
break;
case "create":
byte[] file = File.ReadAllBytes($@"{Directory.GetCurrentDirectory()}/{args[4]}");
var createResp = await bucket.CreateBlob(file, argv);
Console.WriteLine(JsonConvert.SerializeObject(createResp));
break;
case "del": goto case "delete";
case "delete":
var delResp = await bucket.DeleteBlob(argv);
Console.WriteLine(JsonConvert.SerializeObject(delResp));
break;
case "update":
byte[] updateFile = File.ReadAllBytes($@"{Directory.GetCurrentDirectory()}/{args[4]}");
var updateResp = await bucket.UpdateBlob(updateFile, argv);
Console.WriteLine(JsonConvert.SerializeObject(updateResp));
break;
case "put":
byte[] putFile = File.ReadAllBytes($@"{Directory.GetCurrentDirectory()}/{args[4]}");
var putResp = await bucket.PutBlob(putFile, argv);
Console.WriteLine(JsonConvert.SerializeObject(putResp));
break;
default:
Console.WriteLine("Unknown command!");
break;
}
break;
case "test":
byte[] putFileTest = File.ReadAllBytes($@"{Directory.GetCurrentDirectory()}/{args[3]}");
var putRespTest = await client.PutBlob(putFileTest, new Uri($"https://{LoadConfiguration("secrets.json").DO.region.ToString()}.digitaloceanspaces.com/{args[1]}/{args[2]}"));
Console.WriteLine(JsonConvert.SerializeObject(putRespTest));
break;
default:
Console.WriteLine("Unknown command");
break;
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
Console.WriteLine("Invalid number of arguments");
}
await AsyncMain(arg, fac);
}
internal AmazonS3Client(IObjectClient objectClient, IBucketClient bucketClient, IMultipartClient multipartClient, IMultipartTransfer multipartTransfer, ITransfer transfer, ISignedObjectClient signedObjectClient) : base(objectClient, bucketClient, multipartClient, multipartTransfer, transfer, signedObjectClient)
{
}
protected internal ClientBase(IObjectClient objectClient, IBucketClient bucketClient, IMultipartClient multipartClient, IMultipartTransfer multipartTransfer, ITransfer transfer, ISignedObjectClient signedObjectClient)
{
Client = new SimpleClient(objectClient, bucketClient, multipartClient, multipartTransfer, transfer, signedObjectClient);
}
public GCPBucket(StorageClient client, Bucket bucket, IBucketClient bucketClient)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
_bucketClient = bucketClient ?? throw new ArgumentNullException(nameof(bucketClient));
}
internal GoogleCloudStorageClient(IObjectClient objectClient, IBucketClient bucketClient, IMultipartClient multipartClient, IMultipartTransfer multipartTransfer, ITransfer transfer, ISignedObjectClient signedObjectClient)
: base(objectClient, bucketClient, multipartClient, multipartTransfer, transfer, signedObjectClient)
{
}
