public void AnonymousRoundtrip()
{
var accountString = "BlobEndpoint=http://blobs/";
Assert.AreEqual(accountString, StorageConnectionString.Parse(accountString).ToString(true));
var account = new StorageConnectionString(null, new Uri("http://blobs/"), null, null, null);
this.AccountsAreEqual(account, StorageConnectionString.Parse(account.ToString(true)));
}
/// <summary>
/// Create connection string for azure blob storage
/// </summary>
public static StorageConnectionString ForAzureDataLakeStorageWithSharedKey(this IConnectionStringFactory factory,
string accountName,
string accountKey)
{
var cs = new StorageConnectionString(KnownPrefix.AzureDataLakeGen2);
cs.Parameters[KnownParameter.AccountName] = accountName;
cs.Parameters[KnownParameter.KeyOrPassword] = accountKey;
return(cs);
}
/// <summary>
/// Create connection string for azure blob storage
/// </summary>
public static StorageConnectionString ForAzureBlobStorage(this IConnectionStringFactory factory,
string accountName,
string accountKey)
{
var cs = new StorageConnectionString(KnownPrefix.AzureBlobStorage + "://");
cs.Parameters[KnownParameter.AccountName] = accountName;
cs.Parameters[KnownParameter.KeyOrPassword] = accountKey;
return(cs);
}
private void SetConnectionString(string connection)
{
if (string.IsNullOrWhiteSpace(connection))
{
throw new Exception("Azure connection cannot be null or empty");
}
CosmosStorageAccount = Microsoft.Azure.Cosmos.Table.CloudStorageAccount.Parse(connection);
StorageCredentials = (StorageSharedKeyCredential)StorageConnectionString.Parse(connection).Credentials;
}
/// <summary>
/// Initializes a new instance of the <see cref="ShareServiceClient"/>
/// class.
/// </summary>
/// <param name="connectionString">
/// A connection string includes the authentication information
/// required for your application to access data in an Azure Storage
/// account at runtime.
///
/// For more information, <see href="https://docs.microsoft.com/azure/storage/common/storage-configure-connection-string">Configure Azure Storage connection strings</see>
/// </param>
/// <param name="options">
/// Optional client options that define the transport pipeline
/// policies for authentication, retries, etc., that are applied to
/// every request.
/// </param>
public ShareServiceClient(string connectionString, ShareClientOptions options)
{
options ??= new ShareClientOptions();
var conn = StorageConnectionString.Parse(connectionString);
_uri = conn.FileEndpoint;
_pipeline = options.Build(conn.Credentials);
_version = options.Version;
_clientDiagnostics = new ClientDiagnostics(options);
}
public IBlobStorage?CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix != Prefix)
{
return(null);
}
connectionString.GetRequired("username", true, out string userName);
connectionString.GetRequired("password", true, out string password);
connectionString.GetRequired("bucket", true, out string bucket);
return(new PassiveFtpStorage(new NetworkCredential(userName, password, SelectelFtpDomain), bucket));
}
public void Handles_special_characters(string valueToSave)
{
var cs = new StorageConnectionString("local://");
cs["key"] = valueToSave;
string css = cs.ToString();
cs = new StorageConnectionString(css);
Assert.Equal(valueToSave, cs["key"]);
}
private void ValidateRoundTrip(string connectionString)
{
var parsed = StorageConnectionString.Parse(connectionString);
var reserialized = parsed.ToString(true);
var reparsed = StorageConnectionString.Parse(reserialized);
// make sure it round trips
this.AccountsAreEqual(parsed, reparsed);
}
/// <summary>
/// Initializes a new instance of the <see cref="QueueServiceClient"/>
/// class.
/// </summary>
/// <param name="connectionString">
/// A connection string includes the authentication information
/// required for your application to access data in an Azure Storage
/// account at runtime.
///
/// For more information, see
/// <see href="https://docs.microsoft.com/azure/storage/common/storage-configure-connection-string">
/// Configure Azure Storage connection strings</see>.
/// </param>
/// <param name="options">
/// Optional client options that define the transport pipeline
/// policies for authentication, retries, etc., that are applied to
/// every request.
/// </param>
public QueueServiceClient(string connectionString, QueueClientOptions options)
{
var conn = StorageConnectionString.Parse(connectionString);
_uri = conn.QueueEndpoint;
options ??= new QueueClientOptions();
_pipeline = options.Build(conn.Credentials);
_version = options.Version;
_clientDiagnostics = new ClientDiagnostics(options);
_clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == KnownPrefix.AzureFilesStorage)
{
connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName);
connectionString.GetRequired(KnownParameter.KeyOrPassword, true, out string key);
return(AzureFilesBlobStorage.CreateFromAccountNameAndKey(accountName, key));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "google.storage")
{
connectionString.GetRequired("bucket", true, out string bucketName);
connectionString.GetRequired("cred", true, out string base64EncodedJson);
return(StorageFactory.Blobs.GoogleCloudStorageFromJson(bucketName, base64EncodedJson, true));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == KnownPrefix.Databricks)
{
connectionString.GetRequired("baseUri", true, out string baseUri);
connectionString.GetRequired("token", true, out string token);
return(new DatabricksBlobStorage(baseUri, token));
}
return(null);
}
public IKeyValueStorage CreateKeyValueStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.tables")
{
connectionString.GetRequired("account", true, out string acctountName);
connectionString.GetRequired("key", true, out string key);
return(new AzureTableStorageKeyValueStorage(acctountName, key));
}
return(null);
}
/// <summary>
/// Initializes a new instance of the <see cref="QueueServiceClient"/>
/// class.
/// </summary>
/// <param name="connectionString">
/// A connection string includes the authentication information
/// required for your application to access data in an Azure Storage
/// account at runtime.
///
/// For more information, <see href="https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string"/>.
/// </param>
/// <param name="connectionOptions">
/// Optional connection options that define the transport pipeline
/// policies for authentication, retries, etc., that are applied to
/// every request.
/// </param>
/// <remarks>
/// The credentials on <paramref name="connectionString"/> will override those on <paramref name="connectionOptions"/>.
/// </remarks>
public QueueServiceClient(string connectionString, QueueConnectionOptions connectionOptions = default)
{
var conn = StorageConnectionString.Parse(connectionString);
// TODO: perform a copy of the options instead
var connOptions = connectionOptions ?? new QueueConnectionOptions();
connOptions.Credentials = conn.Credentials;
this.Uri = conn.QueueEndpoint;
this._pipeline = connOptions.Build();
}
internal StorageConnectionString GetConnectionString(
SharedAccessSignatureCredentials credentials = default,
bool includeEndpoint = true)
{
credentials ??= GetAccountSasCredentials();
if (!includeEndpoint)
{
return(new StorageConnectionString(
credentials,
(new Uri(TestConfigDefault.BlobServiceEndpoint), new Uri(TestConfigDefault.BlobServiceSecondaryEndpoint)),
(new Uri(TestConfigDefault.QueueServiceEndpoint), new Uri(TestConfigDefault.QueueServiceSecondaryEndpoint)),
(new Uri(TestConfigDefault.TableServiceEndpoint), new Uri(TestConfigDefault.TableServiceSecondaryEndpoint)),
(new Uri(TestConfigDefault.FileServiceEndpoint), new Uri(TestConfigDefault.FileServiceSecondaryEndpoint))));
}
(Uri, Uri)queueUri = (new Uri(TestConfigDefault.QueueServiceEndpoint), new Uri(TestConfigDefault.QueueServiceSecondaryEndpoint));
return(new StorageConnectionString(
credentials,
queueStorageUri: queueUri));
}
public IMessenger CreateMessenger(StorageConnectionString connectionString)
{
if (connectionString.Prefix == Constants.AzureQueueConnectionPrefix)
{
connectionString.GetRequired(KnownParameter.AccountName, true, out string accountName);
connectionString.GetRequired(KnownParameter.KeyOrPassword, true, out string key);
return(new AzureStorageQueueMessenger(accountName, key));
}
return(null);
}
public void DevStoreProxyUri()
{
Assert.IsTrue(StorageConnectionString.TryParse("UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://ipv4.fiddler", out StorageConnectionString devstoreAccount));
Assert.AreEqual(new Uri("http://ipv4.fiddler:10000/devstoreaccount1"), devstoreAccount.BlobEndpoint);
Assert.AreEqual(new Uri("http://ipv4.fiddler:10001/devstoreaccount1"), devstoreAccount.QueueEndpoint);
Assert.AreEqual(new Uri("http://ipv4.fiddler:10000/devstoreaccount1"), devstoreAccount.BlobStorageUri.PrimaryUri);
Assert.AreEqual(new Uri("http://ipv4.fiddler:10001/devstoreaccount1"), devstoreAccount.QueueStorageUri.PrimaryUri);
Assert.AreEqual(new Uri("http://ipv4.fiddler:10000/devstoreaccount1-secondary"), devstoreAccount.BlobStorageUri.SecondaryUri);
Assert.AreEqual(new Uri("http://ipv4.fiddler:10001/devstoreaccount1-secondary"), devstoreAccount.QueueStorageUri.SecondaryUri);
Assert.IsNull(devstoreAccount.FileStorageUri.PrimaryUri);
Assert.IsNull(devstoreAccount.FileStorageUri.SecondaryUri);
}
public IMessenger CreateMessenger(StorageConnectionString connectionString)
{
if (connectionString.Prefix == KnownPrefix.AzureEventHub)
{
if (connectionString.IsNative)
{
return(new AzureEventHubMessenger(connectionString.Native, null));
}
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.blob")
{
connectionString.GetRequired("account", true, out string accountName);
connectionString.GetRequired("key", true, out string key);
return(new AzureUniversalBlobStorageProvider(accountName, key));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.datalake.gen1")
{
connectionString.GetRequired("account", true, out string accountName);
connectionString.GetRequired("tenantId", true, out string tenantId);
connectionString.GetRequired("principalId", true, out string principalId);
connectionString.GetRequired("principalSecret", true, out string principalSecret);
int.TryParse(connectionString.Get("listBatchSize"), out int listBatchSize);
AzureDataLakeGen1Storage client = AzureDataLakeGen1Storage.CreateByClientSecret(
accountName, tenantId, principalId, principalSecret);
if (listBatchSize != 0)
{
client.ListBatchSize = listBatchSize;
}
return(client);
}
else if (connectionString.Prefix == "azure.datalake.gen2")
{
connectionString.GetRequired("account", true, out string accountName);
if (connectionString.Parameters.ContainsKey("msi"))
{
return(AzureDataLakeStoreGen2BlobStorageProvider.CreateByManagedIdentity(accountName));
}
string key = connectionString.Get("key");
if (!string.IsNullOrWhiteSpace(key))
{
//connect with shared key
return(AzureDataLakeStoreGen2BlobStorageProvider.CreateBySharedAccessKey(accountName, key));
}
else
{
//connect with service principal
connectionString.GetRequired("tenantId", true, out string tenantId);
connectionString.GetRequired("principalId", true, out string principalId);
connectionString.GetRequired("principalSecret", true, out string principalSecret);
return(AzureDataLakeStoreGen2BlobStorageProvider.CreateByClientSecret(accountName, tenantId, principalId, principalSecret));
}
}
return(null);
}
public IMessagePublisher CreateMessagePublisher(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.queue")
{
connectionString.GetRequired("account", true, out string accountName);
connectionString.GetRequired("key", true, out string key);
connectionString.GetRequired("queue", true, out string queueName);
return(new AzureStorageQueuePublisher(accountName, key, queueName));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "ftp")
{
connectionString.GetRequired("host", true, out string host);
connectionString.GetRequired("user", true, out string user);
connectionString.GetRequired("password", true, out string password);
return(new FluentFtpBlobStorage(host, new NetworkCredential(user, password)));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.keyvault")
{
connectionString.GetRequired("vaultUri", true, out string uri);
connectionString.GetRequired("clientId", true, out string clientId);
connectionString.GetRequired("clientSecret", true, out string clientSecret);
return(new AzureKeyVaultBlobStorageProvider(new Uri(uri), clientId, clientSecret));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == KnownPrefix.DatabricksDbfs)
{
connectionString.GetRequired("baseUri", true, out string baseUri);
connectionString.GetRequired("token", true, out string token);
string isReadOnlyString = connectionString.Get("isReadOnly");
bool.TryParse(isReadOnlyString, out bool isReadOnly);
return(new AzureDatabricksDbfsBlobStorage(baseUri, token, isReadOnly));
}
return(null);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.blob")
{
connectionString.GetRequired("account", true, out string accountName);
string containerName = connectionString.Get("container");
connectionString.GetRequired("key", true, out string key);
return(new AzureUniversalBlobStorageProvider(accountName, key,
string.IsNullOrEmpty(containerName) ? null : containerName));
}
return(null);
}
/// <summary>
///
/// </summary>
public static StorageConnectionString ForAzureDataLakeStorageWithAzureAd(this IConnectionStringFactory factory,
string accountName,
string tenantId,
string applicationId,
string applicationSecret)
{
var cs = new StorageConnectionString(KnownPrefix.AzureDataLakeGen2);
cs.Parameters[KnownParameter.AccountName] = accountName;
cs.Parameters[KnownParameter.TenantId] = tenantId;
cs.Parameters[KnownParameter.ClientId] = applicationId;
cs.Parameters[KnownParameter.ClientSecret] = applicationSecret;
return(cs);
}
public IBlobStorage CreateBlobStorage(StorageConnectionString connectionString)
{
if (connectionString.Prefix == "azure.datalakestore")
{
connectionString.GetRequired("accountName", true, out string accountName);
connectionString.GetRequired("tenantId", true, out string tenantId);
connectionString.GetRequired("principalId", true, out string principalId);
connectionString.GetRequired("principalSecret", true, out string principalSecret);
return(AzureDataLakeStoreBlobStorageProvider.CreateByClientSecret(accountName, tenantId, principalId, principalSecret));
}
return(null);
}
internal StorageConnectionString GetConnectionString(
SharedAccessSignatureCredentials credentials = default,
bool includeEndpoint = true)
{
credentials ??= GetAccountSasCredentials();
if (!includeEndpoint)
{
return(TestExtensions.CreateStorageConnectionString(
credentials,
TestConfigDefault.AccountName));
}
(Uri, Uri)fileUri = StorageConnectionString.ConstructFileEndpoint(
Constants.Https,
TestConfigDefault.AccountName,
public void ExportKey()
{
var accountKeyString = "abc2564=";
var accountString = "BlobEndpoint=http://blobs/;AccountName=test;AccountKey=" + accountKeyString;
var account = StorageConnectionString.Parse(accountString);
var accountAndKey = (SharedKeyCredentials)account.Credentials;
var key = accountAndKey.ExportBase64EncodedKey();
Assert.AreEqual(accountKeyString, key);
var keyBytes = accountAndKey.AccountKeyValue;
var expectedKeyBytes = Convert.FromBase64String(accountKeyString);
TestHelper.AssertSequenceEqual(expectedKeyBytes, keyBytes);
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
*/
private void DeleteBlobFromAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
IsDeleted.Set(context, AzureHelper.DeleteBlobFromAzureStorage(BlobReference.Get(context)));
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
