public IMessageReceiver CreateMessageReceiver(StorageConnectionString connectionString)
{
if (connectionString.Prefix == Constants.AzureQueueConnectionPrefix)
{
connectionString.GetRequired(Constants.QueueParam, true, out string queueName);
string invisibilityString = connectionString.Get(Constants.InvisibilityParam);
string pollingTimeoutString = connectionString.Get(Constants.PollParam);
if (!TimeSpan.TryParse(invisibilityString, out TimeSpan invisibility))
{
invisibility = TimeSpan.FromMinutes(1);
}
if (!TimeSpan.TryParse(pollingTimeoutString, out TimeSpan polling))
{
polling = TimeSpan.FromMinutes(1);
}
if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) &&
useDevelopment)
{
return(new AzureStorageQueueReceiver(queueName, invisibility, polling));
}
else
{
connectionString.GetRequired(AccountParam, true, out string accountName);
connectionString.GetRequired(KeyParam, true, out string key);
return(new AzureStorageQueueReceiver(accountName, key, queueName, invisibility, polling));
}
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "aws.s3")
{
string keyId = connectionString.Get("keyId");
string key = connectionString.Get("key");
if (string.IsNullOrEmpty(keyId) != string.IsNullOrEmpty(key))
{
throw new ArgumentException($"connection string requires both 'key' and 'keyId' parameters, or neither.");
}
connectionString.GetRequired("bucket", true, out string bucket);
string region = connectionString.Get("region");
RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(string.IsNullOrEmpty(region) ? "eu-west-1" : region);
if (string.IsNullOrEmpty(keyId))
{
return(new AwsS3BlobStorage(bucket, endpoint));
}
return(new AwsS3BlobStorage(keyId, key, bucket, endpoint));
}
return(null);
}
public IMessageReceiver CreateMessageReceiver(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.queue")
{
connectionString.GetRequired("account", true, out string accountName);
connectionString.GetRequired("key", true, out string key);
connectionString.GetRequired("queue", true, out string queueName);
string invisibilityString = connectionString.Get("invisibility");
string pollingTimeoutString = connectionString.Get("poll");
if (!TimeSpan.TryParse(invisibilityString, out TimeSpan invisibility))
{
invisibility = TimeSpan.FromMinutes(1);
}
if (!TimeSpan.TryParse(pollingTimeoutString, out TimeSpan polling))
{
polling = TimeSpan.FromMinutes(1);
}
return(new AzureStorageQueueReceiver(accountName, key, queueName, invisibility, polling));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == KnownPrefix.AzureBlobStorage)
{
if (connectionString.Parameters.ContainsKey(KnownParameter.IsLocalEmulator))
{
return(StorageFactory.Blobs.AzureBlobStorageWithLocalEmulator());
}
connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName);
string sharedKey = connectionString.Get(KnownParameter.KeyOrPassword);
if (!string.IsNullOrEmpty(sharedKey))
{
return(StorageFactory.Blobs.AzureBlobStorageWithSharedKey(accountName, sharedKey));
}
string tenantId = connectionString.Get(KnownParameter.TenantId);
if (!string.IsNullOrEmpty(tenantId))
{
connectionString.GetRequired(KnownParameter.ClientId, true, out string clientId);
connectionString.GetRequired(KnownParameter.ClientSecret, true, out string clientSecret);
return(StorageFactory.Blobs.AzureBlobStorageWithAzureAd(accountName, tenantId, clientId, clientSecret));
}
if (connectionString.Parameters.ContainsKey(KnownParameter.MsiEnabled))
{
return(StorageFactory.Blobs.AzureBlobStorageWithMsi(accountName));
}
}
else if (connectionString.Prefix == KnownPrefix.AzureDataLakeGen2 || connectionString.Prefix == KnownPrefix.AzureDataLake)
{
connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName);
string sharedKey = connectionString.Get(KnownParameter.KeyOrPassword);
if (!string.IsNullOrEmpty(sharedKey))
{
return(StorageFactory.Blobs.AzureDataLakeStorageWithSharedKey(accountName, sharedKey));
}
string tenantId = connectionString.Get(KnownParameter.TenantId);
if (!string.IsNullOrEmpty(tenantId))
{
connectionString.GetRequired(KnownParameter.ClientId, true, out string clientId);
connectionString.GetRequired(KnownParameter.ClientSecret, true, out string clientSecret);
return(StorageFactory.Blobs.AzureDataLakeStorageWithAzureAd(accountName, tenantId, clientId, clientSecret));
}
if (connectionString.Parameters.ContainsKey(KnownParameter.MsiEnabled))
{
return(StorageFactory.Blobs.AzureDataLakeStorageWithMsi(accountName));
}
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.datalake.gen1")
{
connectionString.GetRequired("account", true, out string accountName);
connectionString.GetRequired("tenantId", true, out string tenantId);
connectionString.GetRequired("principalId", true, out string principalId);
connectionString.GetRequired("principalSecret", true, out string principalSecret);
int.TryParse(connectionString.Get("listBatchSize"), out int listBatchSize);
AzureDataLakeGen1Storage client = AzureDataLakeGen1Storage.CreateByClientSecret(
accountName, tenantId, principalId, principalSecret);
if (listBatchSize != 0)
{
client.ListBatchSize = listBatchSize;
}
return(client);
}
else if (connectionString.Prefix == "azure.datalake.gen2")
{
connectionString.GetRequired("account", true, out string accountName);
if (connectionString.Parameters.ContainsKey("msi"))
{
return(AzureDataLakeStoreGen2BlobStorageProvider.CreateByManagedIdentity(accountName));
}
string key = connectionString.Get("key");
if (!string.IsNullOrWhiteSpace(key))
{
//connect with shared key
return(AzureDataLakeStoreGen2BlobStorageProvider.CreateBySharedAccessKey(accountName, key));
}
else
{
//connect with service principal
connectionString.GetRequired("tenantId", true, out string tenantId);
connectionString.GetRequired("principalId", true, out string principalId);
connectionString.GetRequired("principalSecret", true, out string principalSecret);
return(AzureDataLakeStoreGen2BlobStorageProvider.CreateByClientSecret(accountName, tenantId, principalId, principalSecret));
}
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == BlobPrefix)
{
if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) && useDevelopment)
{
return(AzureUniversalBlobStorageProvider.CreateForLocalEmulator());
}
else
{
connectionString.GetRequired(AccountParam, true, out string accountName);
connectionString.GetRequired(KeyParam, true, out string key);
return(AzureUniversalBlobStorageProvider.CreateFromAccountNameAndKey(accountName, key));
}
}
else
{
//try to re-parse native connection string
var newcs = new StorageConnectionString(BlobPrefix + "://" + connectionString.Prefix);
if (newcs.Parameters.TryGetValue("AccountName", out string accountName) &&
newcs.Parameters.TryGetValue("AccountKey", out string accountKey))
{
return(AzureUniversalBlobStorageProvider.CreateFromAccountNameAndKey(accountName, accountKey));
}
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.blob")
{
connectionString.GetRequired("account", true, out string accountName);
string containerName = connectionString.Get("container");
connectionString.GetRequired("key", true, out string key);
return(new AzureUniversalBlobStorageProvider(accountName, key, containerName));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == KnownPrefix.AwsS3)
{
string cliProfileName = connectionString.Get(KnownParameter.LocalProfileName);
connectionString.GetRequired(KnownParameter.BucketName, true, out string bucket);
connectionString.GetRequired(KnownParameter.Region, true, out string region);
if (string.IsNullOrEmpty(cliProfileName))
{
string keyId = connectionString.Get(KnownParameter.KeyId);
string key = connectionString.Get(KnownParameter.KeyOrPassword);
if (string.IsNullOrEmpty(keyId) != string.IsNullOrEmpty(key))
{
throw new ArgumentException($"connection string requires both 'key' and 'keyId' parameters, or neither.");
}
if (string.IsNullOrEmpty(keyId))
{
return(new AwsS3BlobStorage(bucket, region));
}
string sessionToken = connectionString.Get(KnownParameter.SessionToken);
return(new AwsS3BlobStorage(keyId, key, sessionToken, bucket, region, null));
}
#if !NET16
else
{
return(AwsS3BlobStorage.FromAwsCliProfile(cliProfileName, bucket, region));
}
#endif
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == KnownPrefix.DatabricksDbfs)
{
connectionString.GetRequired("baseUri", true, out string baseUri);
connectionString.GetRequired("token", true, out string token);
string isReadOnlyString = connectionString.Get("isReadOnly");
bool.TryParse(isReadOnlyString, out bool isReadOnly);
return(new AzureDatabricksDbfsBlobStorage(baseUri, token, isReadOnly));
}
return(null);
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
* Used storage space size will return in bytes
*/
private void GetUsedSpaceOfAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
UsedSpace.Set(context, AzureHelper.GetUsedSpaceOfAzureStorage(FileExtension.Get(context)));
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
*/
private void CrateContainerInAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
IsContainerCreated.Set(context, AzureHelper.CrateContainerInAzureStorage());
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
* useFlatBlobListing is true to ensure loading all files in virtual blob sub-folders as a plain list
*/
private void GetBlobNamesFromAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
BlobNameList.Set(context, AzureHelper.GetBlobNamesFromAzureStorage(Convert.ToBoolean((int)IsIncludedSubfolder)));
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
*/
private void DeleteBlobFromAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
IsDeleted.Set(context, AzureHelper.DeleteBlobFromAzureStorage(BlobReference.Get(context)));
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
*/
private void UploadToAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
BlobUri.Set(context, AzureHelper.UploadToAzureStorage(SourceFile.Get(context), BlobReference.Get(context), Convert.ToBoolean((int)DeleteSourceAfterUpload)));
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "aws.s3")
{
connectionString.GetRequired("keyId", true, out string keyId);
connectionString.GetRequired("key", true, out string key);
connectionString.GetRequired("bucket", true, out string bucket);
string region = connectionString.Get("region");
RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(string.IsNullOrEmpty(region) ? "eu-west-1" : region);
return(new AwsS3BlobStorageProvider(keyId, key, bucket, endpoint));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.blob")
{
connectionString.GetRequired("account", true, out string accountName);
connectionString.GetRequired("container", true, out string containerName);
connectionString.GetRequired("key", true, out string key);
if (!bool.TryParse(connectionString.Get("createIfNotExists"), out bool createIfNotExists))
{
createIfNotExists = true;
}
return(new AzureBlobStorageProvider(accountName, key, containerName, createIfNotExists));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == Constants.AzureBlobConnectionPrefix)
{
if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) && useDevelopment)
{
return(AzureUniversalBlobStorageProvider.CreateForLocalEmulator());
}
else
{
connectionString.GetRequired(Constants.AccountParam, true, out string accountName);
connectionString.GetRequired(Constants.KeyParam, true, out string key);
return(AzureUniversalBlobStorageProvider.CreateFromAccountNameAndKey(accountName, key));
}
}
return(null);
}
public IKeyValueStorage CreateKeyValueStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == Constants.AzureTablesConnectionPrefix)
{
if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) &&
useDevelopment)
{
return(new AzureTableStorageKeyValueStorage());
}
else
{
connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName);
connectionString.GetRequired(KnownParameter.KeyOrPassword, true, out string key);
return(new AzureTableStorageKeyValueStorage(accountName, key));
}
}
return(null);
}
public IMessagePublisher CreateMessagePublisher(StorageConnectionString connectionString)
{
if (connectionString.Prefix == Constants.AzureQueueConnectionPrefix)
{
connectionString.GetRequired(Constants.QueueParam, true, out string queueName);
if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) &&
useDevelopment)
{
return(new AzureStorageQueuePublisher(queueName));
}
else
{
connectionString.GetRequired(AccountParam, true, out string accountName);
connectionString.GetRequired(KeyParam, true, out string key);
return(new AzureStorageQueuePublisher(accountName, key, queueName));
}
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.datalake.gen1")
{
connectionString.GetRequired("accountName", true, out string accountName);
connectionString.GetRequired("tenantId", true, out string tenantId);
connectionString.GetRequired("principalId", true, out string principalId);
connectionString.GetRequired("principalSecret", true, out string principalSecret);
int.TryParse(connectionString.Get("listBatchSize"), out int listBatchSize);
AzureDataLakeGen1Storage client = AzureDataLakeGen1Storage.CreateByClientSecret(
accountName, tenantId, principalId, principalSecret);
if (listBatchSize != 0)
{
client.ListBatchSize = listBatchSize;
}
return(client);
}
return(null);
}