public static BlobContainer GetAzureContainer(string containerName)
{
StorageAccountInfo accountInfo = StorageAccountInfo.GetAccountInfoFromConfiguration("BlobStorageEndpoint");
BlobStorage blobStorage = BlobStorage.Create(accountInfo);
//The default timeout of 30 seconds is far too short, make it 6 hours.
blobStorage.Timeout = new TimeSpan(6, 0, 0);
if (String.IsNullOrEmpty(containerName))
{
// Default name for new container; Container names have the same restrictions as DNS names
containerName = String.Format("media{0}{1}", DateTime.Now.Year, DateTime.Now.DayOfYear);
}
else
{
//We have received a path from a media file
containerName = containerName.Substring(0, containerName.IndexOf("/"));
}
BlobContainer container = blobStorage.GetBlobContainer(containerName);
//If the Container already exists, false is returned, so go get it
if (!container.DoesContainerExist())
{
container.CreateContainer(null, ContainerAccessControl.Private);
}
return(container);
}
/// <summary>
/// Returns the Windows Azure table storage from the specified storage account.
/// </summary>
/// <param name="accountName">The storage account name.</param>
/// <returns>An instance of the object supporting storage operations against Windows Azure tables.</returns>
public ICloudTableStorage GetTableStorage(string accountName)
{
Guard.ArgumentNotNull(accountName, "accountName");
var callToken = TraceManager.WorkerRoleComponent.TraceIn(accountName);
try
{
StorageAccountConfigurationSettings storageSettings = this.roleConfigExtension.GetSection <StorageAccountConfigurationSettings>(StorageAccountConfigurationSettings.SectionName);
if (storageSettings != null)
{
StorageAccountInfo storageAccount = storageSettings.Accounts.Get(accountName);
if (storageAccount != null)
{
return(new ReliableCloudTableStorage(storageAccount, GetStorageAccountRetryPolicy(storageAccount)));
}
else
{
throw new ConfigurationErrorsException(String.Format(CultureInfo.InvariantCulture, ExceptionMessages.StorageAccountNotFoundInConfigSource, accountName));
}
}
else
{
throw new ConfigurationErrorsException(String.Format(CultureInfo.InvariantCulture, ExceptionMessages.ConfigSectionNotFoundInConfigSource, StorageAccountConfigurationSettings.SectionName));
}
}
finally
{
TraceManager.WorkerRoleComponent.TraceOut(callToken);
}
}
public WindowsAzureStorageHelper()
{
ContainerName = CONTAINER_NAME;
// Open blob storage.
StorageAccountInfo = StorageAccountInfo.GetDefaultBlobStorageAccountFromConfiguration();
BlobStorageType = BlobStorage.Create(StorageAccountInfo);
BlobStorageType.RetryPolicy = RetryPolicies.RetryN(2, TimeSpan.FromMilliseconds(100));
// Open queue storage.
StorageAccountInfo queueAccount = StorageAccountInfo.GetDefaultQueueStorageAccountFromConfiguration();
QueueStorageType = QueueStorage.Create(queueAccount);
QueueStorageType = QueueStorage.Create(queueAccount);
QueueStorageType.RetryPolicy = RetryPolicies.RetryN(2, TimeSpan.FromMilliseconds(100));
// Open table storage.
StorageAccountInfo tableAccount = StorageAccountInfo.GetDefaultTableStorageAccountFromConfiguration();
TableStorageType = TableStorage.Create(tableAccount);
TableStorageType = TableStorage.Create(tableAccount);
TableStorageType.RetryPolicy = RetryPolicies.RetryN(2, TimeSpan.FromMilliseconds(100));
}
public override void ExecuteCmdlet()
{
if (ParameterSetName.Equals(BlobParameterSetName, StringComparison.InvariantCultureIgnoreCase))
{
var toAdd = new StorageAccountInfo
{
Name = Blob,
Properties = new StorageAccountProperties
{
AccessKey = AccessKey
}
};
DataLakeAnalyticsClient.SetStorageAccount(ResourceGroupName, Account, toAdd);
}
else if (Default)
{
var toAdd = new DataLakeStoreAccountInfo
{
Name = DataLakeStore
};
DataLakeAnalyticsClient.SetDefaultDataLakeStoreAccount(ResourceGroupName, Account, toAdd);
}
else
{
WriteWarning(Resources.InvalidDataLakeStoreAccountModificationAttempt);
}
}
public override void ExecuteCmdlet()
{
if (ParameterSetName.Equals(BlobParameterSetName, StringComparison.InvariantCultureIgnoreCase))
{
var toAdd = new StorageAccountInfo
{
Name = Blob,
Properties = new StorageAccountProperties
{
AccessKey = AccessKey
}
};
DataLakeAnalyticsClient.AddStorageAccount(ResourceGroupName, Account, toAdd);
}
else
{
var toAdd = new DataLakeStoreAccountInfo
{
Name = DataLakeStore,
Properties = new DataLakeStoreAccountInfoProperties()
};
DataLakeAnalyticsClient.AddDataLakeStoreAccount(ResourceGroupName, Account, toAdd);
if (Default)
{
DataLakeAnalyticsClient.SetDefaultDataLakeStoreAccount(ResourceGroupName, Account, toAdd);
}
}
}
public override void ExecuteCmdlet()
{
if (ParameterSetName.Equals(BlobParameterSetName, StringComparison.InvariantCultureIgnoreCase))
{
var toAdd = new StorageAccountInfo
{
Name = Blob,
Properties = new StorageAccountProperties
{
AccessKey = AccessKey
}
};
DataLakeAnalyticsClient.AddStorageAccount(ResourceGroupName, Account, toAdd);
}
else
{
var toAdd = new DataLakeStoreAccountInfo
{
Name = DataLakeStore,
Properties = new DataLakeStoreAccountInfoProperties()
};
DataLakeAnalyticsClient.AddDataLakeStoreAccount(ResourceGroupName, Account, toAdd);
if (Default)
{
DataLakeAnalyticsClient.SetDefaultDataLakeStoreAccount(ResourceGroupName, Account, toAdd);
}
}
}
public WindowsAzureStorageHelper(string accountName, string accountKey, bool isLocal, string blobEndpointURI, string queueEndpointURI, string tableEndpointURI)
{
ContainerName = CONTAINER_NAME;
StorageAccountInfo = new StorageAccountInfo(new Uri(blobEndpointURI), isLocal, accountName, accountKey);
BlobStorageType = BlobStorage.Create(StorageAccountInfo);
BlobStorageType.RetryPolicy = RetryPolicies.RetryN(2, TimeSpan.FromMilliseconds(100));
Container = BlobStorageType.GetBlobContainer(ContainerName);
//Create the container if it does not exist.
Container.CreateContainer(ContainerMetaData, ContainerAccessControl.Private);
// Open queue storage.
//StorageAccountInfo qaccount = StorageAccountInfo.GetDefaultQueueStorageAccountFromConfiguration();
StorageAccountInfo qaccount = new StorageAccountInfo(new Uri(queueEndpointURI), isLocal, accountName, accountKey);
QueueStorageType = QueueStorage.Create(qaccount);
QueueStorageType = QueueStorage.Create(qaccount);
QueueStorageType.RetryPolicy = RetryPolicies.RetryN(2, TimeSpan.FromMilliseconds(100));
// Open table storage.
//StorageAccountInfo taccount = StorageAccountInfo.GetDefaultTableStorageAccountFromConfiguration();
StorageAccountInfo taccount = new StorageAccountInfo(new Uri(tableEndpointURI), isLocal, accountName, accountKey);
TableStorageType = TableStorage.Create(taccount);
TableStorageType = TableStorage.Create(taccount);
TableStorageType.RetryPolicy = RetryPolicies.RetryN(2, TimeSpan.FromMilliseconds(100));
}
/// <summary>
/// Initializes a new instance of the <see cref="AdlDataSource"/> class.
/// </summary>
/// <param name="dataSourceInfo">The data source information.</param>
public AdlDataSource(StorageAccountInfo dataSourceInfo)
{
Name = dataSourceInfo.Name;
Type = DataLakeAnalyticsEnums.DataSourceType.Blob;
// always false because there is no concept of a default blob storage container for ADL.
IsDefault = false;
}
private BlobContainer GetContainer()
{
BlobStorage blobStorage = BlobStorage.Create(StorageAccountInfo.GetDefaultBlobStorageAccountFromConfiguration());
BlobContainer newContainer = blobStorage.GetBlobContainer(RoleEnvironment.GetConfigurationSettingValue("ContainerName"));
newContainer.CreateContainer(null, ContainerAccessControl.Public);
return(newContainer);
}
internal static string GetInitExceptionDescription(StorageAccountInfo table, StorageAccountInfo blob)
{
StringBuilder builder = new StringBuilder();
builder.Append(GetInitExceptionDescription(table, "table storage configuration"));
builder.Append(GetInitExceptionDescription(blob, "blob storage configuration"));
return(builder.ToString());
}
/// <summary>
/// Initializes a new instance of the <see cref="AdlDataSource"/> class.
/// </summary>
/// <param name="dataSourceInfo">The data source information.</param>
public AdlDataSource(StorageAccountInfo dataSourceInfo)
{
Name = dataSourceInfo.Name;
Type = DataLakeAnalyticsEnums.DataSourceType.Blob;
// always false because there is no concept of a default blob storage container for ADL.
IsDefault = false;
}
/// <summary>
/// Initializes a new instance of the <see cref="XslTransformCloudBlobCacheExtension"/> object using the specified storage account.
/// The storage account will provide a dedicated blob storage container which will be used as a cache.
/// </summary>
/// <param name="storageAccount">The name of the storage account on which the caching blob storage container will be located.</param>
public XslTransformCloudBlobCacheExtension(StorageAccountInfo storageAccount)
{
Guard.ArgumentNotNull(storageAccount, "storageAccount");
Guard.ArgumentNotNullOrEmptyString(storageAccount.AccountName, "storageAccount.AccountName");
this.storageAccount = storageAccount;
this.ContainerName = DefaultContainerName;
}
private static async Task <string> GetStorageAccountKeyAsync(StorageAccountInfo storageAccountInfo)
{
var azureClient = await GetAzureManagementClientAsync();
var storageAccount = await azureClient.WithSubscription(storageAccountInfo.SubscriptionId).StorageAccounts.GetByIdAsync(storageAccountInfo.Id);
return((await storageAccount.GetKeysAsync()).First().Value);
}
public PSStorageAccountInfo(StorageAccountInfo baseAccount) :
base(
baseAccount.Name,
baseAccount.AccessKey,
baseAccount.Id,
baseAccount.Type,
baseAccount.Suffix)
{
}
// *********************************************************
// Enumerate all Blobs in a Specific Blob Container
// *********************************************************
public IEnumerable <Object> ListBlobs(string ContainerName,
string BlobEndPoint, string Account, string SharedKey)
{
StorageAccountInfo AccountInfo = new StorageAccountInfo(new Uri(BlobEndPoint), null, Account, SharedKey);
var blobStore = BlobStorage.Create(AccountInfo);
var container = blobStore.GetBlobContainer(ContainerName);
return(container.ListBlobs(string.Empty, false));
}
public PSStorageAccountInfo(StorageAccountInfo baseAccount) :
base(
baseAccount.Name,
baseAccount.AccessKey,
baseAccount.Id,
baseAccount.Type,
baseAccount.Suffix)
{
Properties = new PSStorageAccountInfoProperties(baseAccount.AccessKey, baseAccount.Suffix);
}
/// <summary>
/// Initializes a new instance of the <see cref="ReliableHybridBlobStorage"/> class using the specified storage account information, caching service endpoint
/// and custom implementation of <see cref="ICloudStorageEntitySerializer"/> interface. Assumes the default use of <see cref="RetryPolicy.DefaultExponential"/>
/// retry policies when accessing storage and caching services.
/// </summary>
/// <param name="storageAccountInfo">The access credentials for Windows Azure storage account.</param>
/// <param name="cacheEndpointInfo">The endpoint details for Windows Azure Caching Service.</param>
/// <param name="dataSerializer">An instance of the component which performs serialization and deserialization of storage objects.</param>
public ReliableHybridBlobStorage(StorageAccountInfo storageAccountInfo, CachingServiceEndpointInfo cacheEndpointInfo, ICloudStorageEntitySerializer dataSerializer)
: this
(
storageAccountInfo,
new RetryPolicy <StorageTransientErrorDetectionStrategy>(RetryPolicy.DefaultClientRetryCount, RetryPolicy.DefaultMinBackoff, RetryPolicy.DefaultMaxBackoff, RetryPolicy.DefaultClientBackoff),
cacheEndpointInfo,
new RetryPolicy <CacheTransientErrorDetectionStrategy>(RetryPolicy.DefaultClientRetryCount, RetryPolicy.DefaultMinBackoff, RetryPolicy.DefaultMaxBackoff, RetryPolicy.DefaultClientBackoff),
dataSerializer
)
{
}
// **************************
// Delete Blob Container
// **************************
public void DeleteContainer(string ContainerName,
string BlobEndPoint, string Account, string SharedKey)
{
StorageAccountInfo AccountInfo =
new StorageAccountInfo(new Uri(BlobEndPoint),
null, Account, SharedKey);
var blobStore = BlobStorage.Create(AccountInfo);
blobStore.GetBlobContainer(ContainerName).DeleteContainer();
}
public override void ExecuteCmdlet()
{
// We only support updates for Storage accounts.
var toAdd = new StorageAccountInfo
{
Name = Blob,
AccessKey = AccessKey
};
DataLakeAnalyticsClient.SetStorageAccount(ResourceGroupName, Account, toAdd);
}
// ************************************
// Get Blob Containers
// ************************************
public IEnumerable <BlobContainer> ListContainers(string BlobEndPoint,
string Account, string SharedKey)
{
StorageAccountInfo AccountInfo =
new StorageAccountInfo(new Uri(BlobEndPoint),
null, Account, SharedKey);
var blobStore = BlobStorage.Create(AccountInfo);
return(blobStore.ListBlobContainers());
}
private RetryPolicy GetStorageAccountRetryPolicy(StorageAccountInfo storageAccount)
{
Guard.ArgumentNotNull(storageAccount, "storageAccount");
if (!String.IsNullOrEmpty(storageAccount.RetryPolicy))
{
return(RetryPolicyFactory.GetRetryPolicy <StorageTransientErrorDetectionStrategy>(storageAccount.RetryPolicy));
}
else
{
return(this.roleConfigExtension.StorageRetryPolicy);
}
}
// ************************************
// Create a new Blob Container
// ************************************
public void CreateContainer(string ContainerName, bool isPublic,
string BlobEndPoint, string Account, string SharedKey)
{
StorageAccountInfo AccountInfo =
new StorageAccountInfo(new Uri(BlobEndPoint),
null, Account, SharedKey);
var blobStore = BlobStorage.Create(AccountInfo);
ContainerAccessControl accessControl = isPublic ?
ContainerAccessControl.Public : ContainerAccessControl.Private;
blobStore.GetBlobContainer(ContainerName).CreateContainer(null, accessControl);
}
/// <summary>
/// Initializes a new instance of the <see cref="ReliableHybridBlobStorage"/> class using the specified storage account information, caching service endpoint, custom retry policies
/// and a custom implementation of <see cref="ICloudStorageEntitySerializer"/> interface.
/// </summary>
/// <param name="storageAccountInfo">The access credentials for Windows Azure storage account.</param>
/// <param name="storageRetryPolicy">The custom retry policy that will ensure reliable access to the underlying blob storage.</param>
/// <param name="cacheEndpointInfo">The endpoint details for Windows Azure Caching Service.</param>
/// <param name="cacheRetryPolicy">The custom retry policy that will ensure reliable access to the Caching Service.</param>
/// <param name="dataSerializer">An instance of the component which performs serialization and deserialization of storage objects.</param>
public ReliableHybridBlobStorage(StorageAccountInfo storageAccountInfo, RetryPolicy storageRetryPolicy, CachingServiceEndpointInfo cacheEndpointInfo, RetryPolicy cacheRetryPolicy, ICloudStorageEntitySerializer dataSerializer)
{
Guard.ArgumentNotNull(storageAccountInfo, "storageAccountInfo");
Guard.ArgumentNotNull(storageRetryPolicy, "storageRetryPolicy");
Guard.ArgumentNotNull(cacheEndpointInfo, "cacheEndpointInfo");
Guard.ArgumentNotNull(cacheRetryPolicy, "cacheRetryPolicy");
Guard.ArgumentNotNull(dataSerializer, "dataSerializer");
this.dataSerializer = dataSerializer;
this.storageList = new List <ICloudBlobStorage>(2);
this.storageList.Add(this.cacheStorage = new ReliableCloudCacheStorage(cacheEndpointInfo, cacheRetryPolicy, dataSerializer));
this.storageList.Add(this.blobStorage = new ReliableCloudBlobStorage(storageAccountInfo, storageRetryPolicy, dataSerializer));
}
public void SetStorageAccount(string resourceGroupName, string accountName, StorageAccountInfo storageToSet)
{
if (string.IsNullOrEmpty(resourceGroupName))
{
resourceGroupName = GetResourceGroupByAccountName(accountName);
}
var storageParams = new AddStorageAccountParameters
{
Properties = storageToSet.Properties
};
_accountClient.Account.UpdateStorageAccount(resourceGroupName, accountName,
storageToSet.Name, storageParams);
}
// *********************************************************
// Delete a Blob in a Specific Blob Container
// *********************************************************
public void DeleteBlob(string ContainerName, string BlobName,
string BlobEndPoint, string Account, string SharedKey)
{
StorageAccountInfo AccountInfo =
new StorageAccountInfo(new Uri(BlobEndPoint), null, Account, SharedKey);
var blobStore = BlobStorage.Create(AccountInfo);
var container = blobStore.GetBlobContainer(ContainerName);
if (container.DoesBlobExist(BlobName))
{
container.DeleteBlob(BlobName);
}
}
public void AddStorageAccount(string resourceGroupName, string accountName, StorageAccountInfo storageToAdd)
{
if (string.IsNullOrEmpty(resourceGroupName))
{
resourceGroupName = GetResourceGroupByAccountName(accountName);
}
var storageParams = new AddStorageAccountParameters
{
AccessKey = storageToAdd.AccessKey
};
_accountClient.StorageAccounts.Add(resourceGroupName, accountName, storageToAdd.Name,
storageParams);
}
public void SetStorageAccount(string resourceGroupName, string accountName, StorageAccountInfo storageToSet)
{
if (string.IsNullOrEmpty(resourceGroupName))
{
resourceGroupName = GetResourceGroupByAccountName(accountName);
}
var storageParams = new UpdateStorageAccountParameters
{
AccessKey = storageToSet.AccessKey
};
_accountClient.StorageAccounts.Update(resourceGroupName, accountName,
storageToSet.Name, storageParams);
}
/// <summary>
/// Initializes a new instance of the reliable Windows Azure Table storage layer connected to the specified storage account,
/// initialized with the specified retry policy and custom serializer.
/// </summary>
/// <param name="storageAccountInfo">The access credentials for Windows Azure storage account.</param>
/// <param name="retryPolicy">The custom retry policy that will ensure reliable access to the underlying table storage.</param>
/// <param name="dataSerializer">An instance of the component which performs serialization and deserialization of storage objects.</param>
public ReliableCloudTableStorage(StorageAccountInfo storageAccountInfo, RetryPolicy retryPolicy, ICloudStorageEntitySerializer dataSerializer)
{
Guard.ArgumentNotNull(storageAccountInfo, "storageAccountInfo");
Guard.ArgumentNotNull(retryPolicy, "retryPolicy");
Guard.ArgumentNotNull(dataSerializer, "dataSerializer");
this.retryPolicy = retryPolicy;
this.dataSerializer = dataSerializer;
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(storageAccountInfo.AccountName, storageAccountInfo.AccountKey), true);
this.tableStorage = storageAccount.CreateCloudTableClient();
// Configure the Table storage not to enforce any retry policies since this is something that we will be dealing ourselves.
this.tableStorage.RetryPolicy = RetryPolicies.NoRetry();
}
// *********************************************************
// Create a new Blob in a Specific Blob Container
// *********************************************************
public void CreateBlob(string ContainerName, string FileName,
string BlobEndPoint, string Account, string SharedKey)
{
StorageAccountInfo AccountInfo =
new StorageAccountInfo(new Uri(BlobEndPoint), null, Account, SharedKey);
var blobStore = BlobStorage.Create(AccountInfo);
var container = blobStore.GetBlobContainer(ContainerName);
BlobProperties props = new BlobProperties(System.IO.Path.GetFileName(FileName));
BlobContents blobContents =
new BlobContents(new System.IO.FileStream(FileName, FileMode.Open));
container.CreateBlob(props, blobContents, true);
}
internal static string GetInitExceptionDescription(StorageAccountInfo info, string desc)
{
StringBuilder builder = new StringBuilder();
builder.Append("The reason for this exception is typically that the endpoints are not correctly configured. " + Environment.NewLine);
if (info == null)
{
builder.Append("The current " + desc + " is null. Please specify a table endpoint!" + Environment.NewLine);
}
else
{
string baseUri = (info.BaseUri == null) ? "null" : info.BaseUri.ToString();
builder.Append("The current " + desc + " is: " + baseUri + ", usePathStyleUris is: " + info.UsePathStyleUris + Environment.NewLine);
builder.Append("Please also make sure that the account name and the shared key are specified correctly. This information cannot be shown here because of security reasons.");
}
return(builder.ToString());
}
private static void ApplicationStartUponFirstRequest(HttpContext context)
{
// This is where you put initialization logic for the site.
// RoleManager is properly initialized at this point.
// Create the tables on first request initialization as there is a performance impact
// if you call CreateTablesFromModel() when the tables already exist. This limits the exposure of
// creating tables multiple times.
// Get the settings from the Service Configuration file
StorageAccountInfo account = StorageAccountInfo.GetDefaultTableStorageAccountFromConfiguration();
// Create the tables
// In this case, just a single table.
// This will create tables for all public properties that are IQueryable (collections)
TableStorage.CreateTablesFromModel(typeof(zlnkDC), account);
}
public async Task GetStorageAccountInfoAsync_UsesCache()
{
var azureProxy = GetMockAzureProxy();
var storageAccountInfo = new StorageAccountInfo {
Name = "defaultstorageaccount", Id = "Id", BlobEndpoint = "https://defaultstorageaccount/", SubscriptionId = "SubId"
};
azureProxy.Setup(a => a.GetStorageAccountInfoAsync(It.IsAny <string>())).Returns(Task.FromResult(storageAccountInfo));
var cachingAzureProxy = new CachingWithRetriesAzureProxy(azureProxy.Object, cache);
var info1 = await cachingAzureProxy.GetStorageAccountInfoAsync("defaultstorageaccount");
var info2 = await cachingAzureProxy.GetStorageAccountInfoAsync("defaultstorageaccount");
azureProxy.Verify(mock => mock.GetStorageAccountInfoAsync("defaultstorageaccount"), Times.Once());
Assert.AreEqual(storageAccountInfo, info1);
Assert.AreEqual(info1, info2);
}
public override void ExecuteCmdlet()
{
if (ParameterSetName.Equals(BlobParameterSetName, StringComparison.InvariantCultureIgnoreCase))
{
var toAdd = new StorageAccountInfo
{
Name = Blob,
Properties = new StorageAccountProperties
{
AccessKey = AccessKey
}
};
DataLakeAnalyticsClient.SetStorageAccount(ResourceGroupName, Account, toAdd);
}
else if (Default)
{
var toAdd = new DataLakeStoreAccountInfo
{
Name = DataLakeStore
};
DataLakeAnalyticsClient.SetDefaultDataLakeStoreAccount(ResourceGroupName, Account, toAdd);
}
else
{
WriteWarning(Resources.InvalidDataLakeStoreAccountModificationAttempt);
}
}
public void SetStorageAccount(string resourceGroupName, string accountName, StorageAccountInfo storageToSet)
{
if (string.IsNullOrEmpty(resourceGroupName))
{
resourceGroupName = GetResourceGroupByAccountName(accountName);
}
var storageParams = new AddStorageAccountParameters
{
Properties = storageToSet.Properties
};
_accountClient.Account.UpdateStorageAccount(resourceGroupName, accountName,
storageToSet.Name, storageParams);
}
