public IMessageReceiver CreateMessageReceiver(StorageConnectionString connectionString) { if (connectionString.Prefix == Constants.AzureQueueConnectionPrefix) { connectionString.GetRequired(Constants.QueueParam, true, out string queueName); string invisibilityString = connectionString.Get(Constants.InvisibilityParam); string pollingTimeoutString = connectionString.Get(Constants.PollParam); if (!TimeSpan.TryParse(invisibilityString, out TimeSpan invisibility)) { invisibility = TimeSpan.FromMinutes(1); } if (!TimeSpan.TryParse(pollingTimeoutString, out TimeSpan polling)) { polling = TimeSpan.FromMinutes(1); } if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) && useDevelopment) { return(new AzureStorageQueueReceiver(queueName, invisibility, polling)); } else { connectionString.GetRequired(AccountParam, true, out string accountName); connectionString.GetRequired(KeyParam, true, out string key); return(new AzureStorageQueueReceiver(accountName, key, queueName, invisibility, polling)); } } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == "aws.s3") { string keyId = connectionString.Get("keyId"); string key = connectionString.Get("key"); if (string.IsNullOrEmpty(keyId) != string.IsNullOrEmpty(key)) { throw new ArgumentException($"connection string requires both 'key' and 'keyId' parameters, or neither."); } connectionString.GetRequired("bucket", true, out string bucket); string region = connectionString.Get("region"); RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(string.IsNullOrEmpty(region) ? "eu-west-1" : region); if (string.IsNullOrEmpty(keyId)) { return(new AwsS3BlobStorage(bucket, endpoint)); } return(new AwsS3BlobStorage(keyId, key, bucket, endpoint)); } return(null); }
public IMessageReceiver CreateMessageReceiver(StorageConnectionString connectionString) { if (connectionString.Prefix == "azure.queue") { connectionString.GetRequired("account", true, out string accountName); connectionString.GetRequired("key", true, out string key); connectionString.GetRequired("queue", true, out string queueName); string invisibilityString = connectionString.Get("invisibility"); string pollingTimeoutString = connectionString.Get("poll"); if (!TimeSpan.TryParse(invisibilityString, out TimeSpan invisibility)) { invisibility = TimeSpan.FromMinutes(1); } if (!TimeSpan.TryParse(pollingTimeoutString, out TimeSpan polling)) { polling = TimeSpan.FromMinutes(1); } return(new AzureStorageQueueReceiver(accountName, key, queueName, invisibility, polling)); } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == KnownPrefix.AzureBlobStorage) { if (connectionString.Parameters.ContainsKey(KnownParameter.IsLocalEmulator)) { return(StorageFactory.Blobs.AzureBlobStorageWithLocalEmulator()); } connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName); string sharedKey = connectionString.Get(KnownParameter.KeyOrPassword); if (!string.IsNullOrEmpty(sharedKey)) { return(StorageFactory.Blobs.AzureBlobStorageWithSharedKey(accountName, sharedKey)); } string tenantId = connectionString.Get(KnownParameter.TenantId); if (!string.IsNullOrEmpty(tenantId)) { connectionString.GetRequired(KnownParameter.ClientId, true, out string clientId); connectionString.GetRequired(KnownParameter.ClientSecret, true, out string clientSecret); return(StorageFactory.Blobs.AzureBlobStorageWithAzureAd(accountName, tenantId, clientId, clientSecret)); } if (connectionString.Parameters.ContainsKey(KnownParameter.MsiEnabled)) { return(StorageFactory.Blobs.AzureBlobStorageWithMsi(accountName)); } } else if (connectionString.Prefix == KnownPrefix.AzureDataLakeGen2 || connectionString.Prefix == KnownPrefix.AzureDataLake) { connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName); string sharedKey = connectionString.Get(KnownParameter.KeyOrPassword); if (!string.IsNullOrEmpty(sharedKey)) { return(StorageFactory.Blobs.AzureDataLakeStorageWithSharedKey(accountName, sharedKey)); } string tenantId = connectionString.Get(KnownParameter.TenantId); if (!string.IsNullOrEmpty(tenantId)) { connectionString.GetRequired(KnownParameter.ClientId, true, out string clientId); connectionString.GetRequired(KnownParameter.ClientSecret, true, out string clientSecret); return(StorageFactory.Blobs.AzureDataLakeStorageWithAzureAd(accountName, tenantId, clientId, clientSecret)); } if (connectionString.Parameters.ContainsKey(KnownParameter.MsiEnabled)) { return(StorageFactory.Blobs.AzureDataLakeStorageWithMsi(accountName)); } } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == "azure.datalake.gen1") { connectionString.GetRequired("account", true, out string accountName); connectionString.GetRequired("tenantId", true, out string tenantId); connectionString.GetRequired("principalId", true, out string principalId); connectionString.GetRequired("principalSecret", true, out string principalSecret); int.TryParse(connectionString.Get("listBatchSize"), out int listBatchSize); AzureDataLakeGen1Storage client = AzureDataLakeGen1Storage.CreateByClientSecret( accountName, tenantId, principalId, principalSecret); if (listBatchSize != 0) { client.ListBatchSize = listBatchSize; } return(client); } else if (connectionString.Prefix == "azure.datalake.gen2") { connectionString.GetRequired("account", true, out string accountName); if (connectionString.Parameters.ContainsKey("msi")) { return(AzureDataLakeStoreGen2BlobStorageProvider.CreateByManagedIdentity(accountName)); } string key = connectionString.Get("key"); if (!string.IsNullOrWhiteSpace(key)) { //connect with shared key return(AzureDataLakeStoreGen2BlobStorageProvider.CreateBySharedAccessKey(accountName, key)); } else { //connect with service principal connectionString.GetRequired("tenantId", true, out string tenantId); connectionString.GetRequired("principalId", true, out string principalId); connectionString.GetRequired("principalSecret", true, out string principalSecret); return(AzureDataLakeStoreGen2BlobStorageProvider.CreateByClientSecret(accountName, tenantId, principalId, principalSecret)); } } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == BlobPrefix) { if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) && useDevelopment) { return(AzureUniversalBlobStorageProvider.CreateForLocalEmulator()); } else { connectionString.GetRequired(AccountParam, true, out string accountName); connectionString.GetRequired(KeyParam, true, out string key); return(AzureUniversalBlobStorageProvider.CreateFromAccountNameAndKey(accountName, key)); } } else { //try to re-parse native connection string var newcs = new StorageConnectionString(BlobPrefix + "://" + connectionString.Prefix); if (newcs.Parameters.TryGetValue("AccountName", out string accountName) && newcs.Parameters.TryGetValue("AccountKey", out string accountKey)) { return(AzureUniversalBlobStorageProvider.CreateFromAccountNameAndKey(accountName, accountKey)); } } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == "azure.blob") { connectionString.GetRequired("account", true, out string accountName); string containerName = connectionString.Get("container"); connectionString.GetRequired("key", true, out string key); return(new AzureUniversalBlobStorageProvider(accountName, key, containerName)); } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == KnownPrefix.AwsS3) { string cliProfileName = connectionString.Get(KnownParameter.LocalProfileName); connectionString.GetRequired(KnownParameter.BucketName, true, out string bucket); connectionString.GetRequired(KnownParameter.Region, true, out string region); if (string.IsNullOrEmpty(cliProfileName)) { string keyId = connectionString.Get(KnownParameter.KeyId); string key = connectionString.Get(KnownParameter.KeyOrPassword); if (string.IsNullOrEmpty(keyId) != string.IsNullOrEmpty(key)) { throw new ArgumentException($"connection string requires both 'key' and 'keyId' parameters, or neither."); } if (string.IsNullOrEmpty(keyId)) { return(new AwsS3BlobStorage(bucket, region)); } string sessionToken = connectionString.Get(KnownParameter.SessionToken); return(new AwsS3BlobStorage(keyId, key, sessionToken, bucket, region, null)); } #if !NET16 else { return(AwsS3BlobStorage.FromAwsCliProfile(cliProfileName, bucket, region)); } #endif } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == KnownPrefix.DatabricksDbfs) { connectionString.GetRequired("baseUri", true, out string baseUri); connectionString.GetRequired("token", true, out string token); string isReadOnlyString = connectionString.Get("isReadOnly"); bool.TryParse(isReadOnlyString, out bool isReadOnly); return(new AzureDatabricksDbfsBlobStorage(baseUri, token, isReadOnly)); } return(null); }
/* * If ContinueOnError argument is 'true', return the exception else nope. * Used storage space size will return in bytes */ private void GetUsedSpaceOfAzureStorage(CodeActivityContext context) { try { AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context)); UsedSpace.Set(context, AzureHelper.GetUsedSpaceOfAzureStorage(FileExtension.Get(context))); } catch (Exception ex) { if (Convert.ToBoolean((int)ContinueOnError)) { } else { throw ex; } } }
/* * If ContinueOnError argument is 'true', return the exception else nope. */ private void CrateContainerInAzureStorage(CodeActivityContext context) { try { AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context)); IsContainerCreated.Set(context, AzureHelper.CrateContainerInAzureStorage()); } catch (Exception ex) { if (Convert.ToBoolean((int)ContinueOnError)) { } else { throw ex; } } }
/* * If ContinueOnError argument is 'true', return the exception else nope. * useFlatBlobListing is true to ensure loading all files in virtual blob sub-folders as a plain list */ private void GetBlobNamesFromAzureStorage(CodeActivityContext context) { try { AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context)); BlobNameList.Set(context, AzureHelper.GetBlobNamesFromAzureStorage(Convert.ToBoolean((int)IsIncludedSubfolder))); } catch (Exception ex) { if (Convert.ToBoolean((int)ContinueOnError)) { } else { throw ex; } } }
/* * If ContinueOnError argument is 'true', return the exception else nope. */ private void DeleteBlobFromAzureStorage(CodeActivityContext context) { try { AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context)); IsDeleted.Set(context, AzureHelper.DeleteBlobFromAzureStorage(BlobReference.Get(context))); } catch (Exception ex) { if (Convert.ToBoolean((int)ContinueOnError)) { } else { throw ex; } } }
/* * If ContinueOnError argument is 'true', return the exception else nope. */ private void UploadToAzureStorage(CodeActivityContext context) { try { AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context)); BlobUri.Set(context, AzureHelper.UploadToAzureStorage(SourceFile.Get(context), BlobReference.Get(context), Convert.ToBoolean((int)DeleteSourceAfterUpload))); } catch (Exception ex) { if (Convert.ToBoolean((int)ContinueOnError)) { } else { throw ex; } } }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == "aws.s3") { connectionString.GetRequired("keyId", true, out string keyId); connectionString.GetRequired("key", true, out string key); connectionString.GetRequired("bucket", true, out string bucket); string region = connectionString.Get("region"); RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(string.IsNullOrEmpty(region) ? "eu-west-1" : region); return(new AwsS3BlobStorageProvider(keyId, key, bucket, endpoint)); } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == "azure.blob") { connectionString.GetRequired("account", true, out string accountName); connectionString.GetRequired("container", true, out string containerName); connectionString.GetRequired("key", true, out string key); if (!bool.TryParse(connectionString.Get("createIfNotExists"), out bool createIfNotExists)) { createIfNotExists = true; } return(new AzureBlobStorageProvider(accountName, key, containerName, createIfNotExists)); } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == Constants.AzureBlobConnectionPrefix) { if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) && useDevelopment) { return(AzureUniversalBlobStorageProvider.CreateForLocalEmulator()); } else { connectionString.GetRequired(Constants.AccountParam, true, out string accountName); connectionString.GetRequired(Constants.KeyParam, true, out string key); return(AzureUniversalBlobStorageProvider.CreateFromAccountNameAndKey(accountName, key)); } } return(null); }
public IKeyValueStorage CreateKeyValueStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == Constants.AzureTablesConnectionPrefix) { if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) && useDevelopment) { return(new AzureTableStorageKeyValueStorage()); } else { connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName); connectionString.GetRequired(KnownParameter.KeyOrPassword, true, out string key); return(new AzureTableStorageKeyValueStorage(accountName, key)); } } return(null); }
public IMessagePublisher CreateMessagePublisher(StorageConnectionString connectionString) { if (connectionString.Prefix == Constants.AzureQueueConnectionPrefix) { connectionString.GetRequired(Constants.QueueParam, true, out string queueName); if (bool.TryParse(connectionString.Get(Constants.UseDevelopmentStorage), out bool useDevelopment) && useDevelopment) { return(new AzureStorageQueuePublisher(queueName)); } else { connectionString.GetRequired(AccountParam, true, out string accountName); connectionString.GetRequired(KeyParam, true, out string key); return(new AzureStorageQueuePublisher(accountName, key, queueName)); } } return(null); }
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString) { if (connectionString.Prefix == "azure.datalake.gen1") { connectionString.GetRequired("accountName", true, out string accountName); connectionString.GetRequired("tenantId", true, out string tenantId); connectionString.GetRequired("principalId", true, out string principalId); connectionString.GetRequired("principalSecret", true, out string principalSecret); int.TryParse(connectionString.Get("listBatchSize"), out int listBatchSize); AzureDataLakeGen1Storage client = AzureDataLakeGen1Storage.CreateByClientSecret( accountName, tenantId, principalId, principalSecret); if (listBatchSize != 0) { client.ListBatchSize = listBatchSize; } return(client); } return(null); }