/// <summary>
/// Download a package from blob storage and unzip it
/// </summary>
/// <param name="containerName">The Blob storage container name</param>
/// <param name="packageName">The name of the zip file package</param>
/// <param name="workingDirectory">Where to extract the files</param>
private static void InstallPackage(string containerName, string packageName, string workingDirectory)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue(DATA_CONNECTION_STRING));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
blobClient.RetryPolicy = RetryPolicies.Retry(100, TimeSpan.FromSeconds(1));
blobClient.Timeout = TimeSpan.FromSeconds(600);
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(packageName);
Console.WriteLine(string.Format("Downloading {0} to {1}", blob.Uri, workingDirectory), "Information");
if (blob.Uri.PathAndQuery.EndsWith(".zip", true, Thread.CurrentThread.CurrentCulture))
{
// if this is a zip file, unzip it
// the temp dirtectory is limited to 100 MB
var filename = @"c:\" + Guid.NewGuid();
blob.DownloadToFile(filename);
Console.WriteLine(string.Format("Extracting {0}", packageName), "Information");
UnZip(Directory.GetCurrentDirectory(), filename, workingDirectory);
// delete the temp file
File.Delete(filename);
}
else
{
blob.DownloadToFile(@"c:\Applications\" + blob.Uri.Segments.Last());
}
Console.WriteLine("Extraction finished", "Information");
}
/// <summary>
/// Checks a package in Blob Storage against any previous package receipt
/// to determine whether to reinstall it
/// </summary>
private static bool IsNewPackage(string containerName, string packageName, string packageReceiptFile)
{
var storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue(DATA_CONNECTION_STRING));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
blobClient.RetryPolicy = RetryPolicies.Retry(100, TimeSpan.FromSeconds(1));
blobClient.Timeout = TimeSpan.FromSeconds(600);
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(packageName);
blob.FetchAttributes();
DateTime blobTimeStamp = blob.Attributes.Properties.LastModifiedUtc;
DateTime fileTimeStamp = File.GetCreationTimeUtc(packageReceiptFile);
if (fileTimeStamp.CompareTo(blobTimeStamp) < 0)
{
Console.WriteLine(string.Format("{0} is new or not yet installed.", packageName), "Information");
return(true);
}
else
{
Console.WriteLine(string.Format("{0} has previously been installed, skipping download.", packageName), "Information");
return(false);
}
}
public static void Initialize()
{
CloudStorageAccount account = CloudConfiguration.GetStorageAccount(AzureConnectionStrings.DataConnection);
// Tables
var cloudTableClient = new CloudTableClient(account.TableEndpoint.ToString(), account.Credentials);
cloudTableClient.CreateTableIfNotExist <ExpenseExpenseItemEntity>(AzureStorageNames.ExpenseTable);
cloudTableClient.CreateTableIfNotExist <ExpenseExportEntity>(AzureStorageNames.ExpenseExportTable);
// Blobs
CloudBlobClient client = account.CreateCloudBlobClient();
client.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(5));
var container = client.GetContainerReference(AzureStorageNames.ReceiptContainerName);
container.CreateIfNotExist();
container = client.GetContainerReference(AzureStorageNames.ExpenseExportContainerName);
container.CreateIfNotExist();
// Queues
CloudQueueClient queueClient = account.CreateCloudQueueClient();
queueClient.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(5));
CloudQueue queueReference = queueClient.GetQueueReference(AzureStorageNames.ApprovedExpenseMessage);
queueReference.CreateIfNotExist();
queueReference = queueClient.GetQueueReference(AzureStorageNames.PoisonApprovedExpenseMessage);
queueReference.CreateIfNotExist();
queueReference = queueClient.GetQueueReference(AzureStorageNames.NewReceiptMessage);
queueReference.CreateIfNotExist();
queueReference = queueClient.GetQueueReference(AzureStorageNames.PoisonNewReceiptMessage);
queueReference.CreateIfNotExist();
}
// Constructor - get settings from a hosted service configuration or .NET configuration file.
public BlobHelper(string configurationSettingName, bool hostedService)
{
if (hostedService)
{
CloudStorageAccount.SetConfigurationSettingPublisher(
(configName, configSettingPublisher) =>
{
var connectionString = RoleEnvironment.GetConfigurationSettingValue(configName);
configSettingPublisher(connectionString);
}
);
}
else
{
CloudStorageAccount.SetConfigurationSettingPublisher(
(configName, configSettingPublisher) =>
{
var connectionString = ConfigurationManager.ConnectionStrings[configName].ConnectionString;
configSettingPublisher(connectionString);
}
);
}
Account = CloudStorageAccount.FromConfigurationSetting(configurationSettingName);
BlobClient = Account.CreateCloudBlobClient();
BlobClient.RetryPolicy = RetryPolicies.Retry(4, TimeSpan.Zero);
}
public DatasetInfoDataSource()
{
_context = new DatasetInfoDataContext(StorageAccount.TableEndpoint.AbsoluteUri, StorageAccount.Credentials)
{
RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1))
};
}
static AzureQueueDefaultPolicies()
{
MaxQueueOperationRetries = 5;
PauseBetweenQueueOperationRetries = TimeSpan.FromMilliseconds(100);
QueueOperationRetryPolicy = RetryPolicies.Retry(MaxQueueOperationRetries, PauseBetweenQueueOperationRetries); // 5 x 100ms
QueueOperationTimeout = PauseBetweenQueueOperationRetries.Multiply(MaxQueueOperationRetries).Multiply(6); // 3 sec
}
public void InitializeCloudStorageAccess(string containerName)
{
if (containerName == null)
{
throw new ArgumentNullException("containerName");
}
if (containerName == "")
{
throw new ArgumentException("Invalid container name", "containerName");
}
if (initializedContainerName != null)
{
if (containerName == initializedContainerName)
{
return;
}
if (containerName != initializedContainerName)
{
throw new NotSupportedException("InitializeCloudStorageAccess already initialized with container name: "
+ initializedContainerName + " (tried to initialize with: "
+ containerName + ")");
}
}
CloudBlobClient blobClient = StorageSupport.CurrStorageAccount.CreateCloudBlobClient();
blobClient.RetryPolicy = RetryPolicies.Retry(10, TimeSpan.FromMilliseconds(300));
CurrBlobClient = blobClient;
var activeContainer = blobClient.GetContainerReference(containerName.ToLower());
activeContainer.CreateIfNotExist();
CurrActiveContainer = activeContainer;
initializedContainerName = containerName;
}
private void Connect()
{
connected = false;
int trycount = 0;
while (trycount++ < 3)
{
try
{
string connectionString = RoleEnvironment.GetConfigurationSettingValue(connectionStringName);
_storageAccount = CloudStorageAccount.Parse(connectionString);
_tableClient = new CloudTableClient(_storageAccount.TableEndpoint.AbsoluteUri, _storageAccount.Credentials);
_tableClient.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1));
if (!_tableClient.DoesTableExist(tableName))
{
_tableClient.CreateTableIfNotExist(tableName);
}
connected = true;
break;
}
catch
{
}
}
if (!connected)
{
throw new Exception("Could not connect to table service");
}
}
// Constructor - pass in a storage connection string.
public BlobHelper(string connectionString)
{
Account = CloudStorageAccount.Parse(connectionString);
BlobClient = Account.CreateCloudBlobClient();
BlobClient.RetryPolicy = RetryPolicies.Retry(4, TimeSpan.Zero);
}
static AzureTableDefaultPolicies()
{
MaxTableCreationRetries = 60;
PauseBetweenTableCreationRetries = TimeSpan.FromSeconds(1);
MaxTableOperationRetries = 5;
PauseBetweenTableOperationRetries = TimeSpan.FromMilliseconds(100);
MaxBusyRetries = 120;
PauseBetweenBusyRetries = TimeSpan.FromMilliseconds(500);
#if DEBUG
if (Debugger.IsAttached)
{
PauseBetweenTableCreationRetries = PauseBetweenTableCreationRetries.Multiply(100);
PauseBetweenTableOperationRetries = PauseBetweenTableOperationRetries.Multiply(100);
PauseBetweenBusyRetries = PauseBetweenBusyRetries.Multiply(10);
}
#endif
TableCreationRetryPolicy = RetryPolicies.Retry(MaxTableCreationRetries, PauseBetweenTableCreationRetries); // 60 x 1s
TableCreationTimeout = PauseBetweenTableCreationRetries.Multiply(MaxTableCreationRetries).Multiply(3); // 3 min
TableOperationRetryPolicy = RetryPolicies.Retry(MaxTableOperationRetries, PauseBetweenTableOperationRetries); // 5 x 100ms
TableOperationTimeout = PauseBetweenTableOperationRetries.Multiply(MaxTableOperationRetries).Multiply(6); // 3 sec
BusyRetriesTimeout = PauseBetweenBusyRetries.Multiply(MaxBusyRetries); // 1 minute
}
private CloudBlobContainer GetBlobContainer(string containerName)
{
var client = this.StorageAccount.CreateCloudBlobClient();
client.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(5));
return(client.GetContainerReference(containerName));
}
public AzureFilesBlobContainer()
{
_log = ClassLogger.Create(GetType());
_dblog = DebugOnlyLogger.Create(_log);
var config = Catalog.Factory.Resolve <IConfig>(SpecialFactoryContexts.Routed);
_containerName = config[BlobContainerLocalConfig.ContainerName];
EntityAccess access = (EntityAccess)Enum.Parse(typeof(EntityAccess),
config.Get(BlobContainerLocalConfig.OptionalAccess,
EntityAccess.Private.ToString()));
_contentType = config.Get(BlobContainerLocalConfig.OptionalContentType, "application/raw");
_account = Client.FromConfig();
_client = _account.CreateCloudBlobClient();
_client.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(5));
var blobContainerPermissions = new BlobContainerPermissions
{
PublicAccess = AzureEntityAccessTranslator.Translate(access)
};
_blobContainerPermissions = blobContainerPermissions;
_container = _client.GetContainerReference(_containerName.ToLowerInvariant());
if (_container.CreateIfNotExist())
{
_container.SetPermissions(_blobContainerPermissions);
}
}
private void Connect()
{
connected = false;
int trycount = 0;
while (trycount++ < 3)
{
try
{
//comment added just for test
string connectionString = "DefaultEndpointsProtocol=https; AccountName=careerthesaurus; AccountKey=52D4zWvYaIL6gfl4FUt6y8cc9Ar8UV8EWNmBttpraVkMJjcy+cOlDhiZYTnmGZLdpKV1nwlNNlGQJiqZh9rlxQ==";
_storageAccount = CloudStorageAccount.Parse(connectionString);
_tableClient = new CloudTableClient(_storageAccount.TableEndpoint.AbsoluteUri, _storageAccount.Credentials);
_tableClient.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1));
if (!_tableClient.DoesTableExist(tableName))
{
_tableClient.CreateTableIfNotExist(tableName);
}
connected = true;
break;
}
catch
{
}
}
if (!connected)
{
throw new Exception("Could not connect to table service");
}
}
private void Init(string configurationSettingName)
{
if (RoleEnvironment.IsAvailable)
{
CloudStorageAccount.SetConfigurationSettingPublisher(
(configName, configSettingPublisher) =>
{
var connectionString = RoleEnvironment.GetConfigurationSettingValue(configName);
configSettingPublisher(connectionString);
}
);
}
else
{
CloudStorageAccount.SetConfigurationSettingPublisher(
(configName, configSettingPublisher) =>
{
var connectionString = ConfigurationManager.ConnectionStrings[configName].ConnectionString;
configSettingPublisher(connectionString);
}
);
}
Account = CloudStorageAccount.FromConfigurationSetting(configurationSettingName);
TableClient = Account.CreateCloudTableClient();
TableClient.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromMilliseconds(100));
}
public GuestBookDataSource()
{
context = new GuestBookDataContext(storageAccount.TableEndpoint.AbsoluteUri, storageAccount.Credentials)
{
RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1))
};
}
/// <summary>
/// Download a package from blob storage and unzip it
/// </summary>
/// <param name="containerName">The Blob storage container name</param>
/// <param name="packageName">The name of the zip file package</param>
/// <param name="workingDirectory">Where to extract the files</param>
private static void InstallPackage(string containerName, string packageName, string workingDirectory)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue(DATA_CONNECTION_STRING));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
blobClient.RetryPolicy = RetryPolicies.Retry(100, TimeSpan.FromSeconds(1));
blobClient.Timeout = TimeSpan.FromSeconds(600);
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(packageName);
Console.WriteLine(string.Format("Downloading {0} to {1}", blob.Uri, workingDirectory), "Information");
string filename = Path.GetTempFileName();
blob.DownloadToFile(filename);
Console.WriteLine(string.Format("Extracting {0}", packageName), "Information");
UnZip(Directory.GetCurrentDirectory(), filename, workingDirectory);
// delete the temp file
File.Delete(filename);
Console.WriteLine("Extraction finished", "Information");
}
public BlobRepository()
{
var connectionString = System.Configuration.ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString;
StorageAccount = CloudStorageAccount.Parse(connectionString);
BlobClient = StorageAccount.CreateCloudBlobClient();
BlobClient.RetryPolicy = RetryPolicies.Retry(4, TimeSpan.Zero);
}
public void AddEventMessage(EventMessage msg)
{
this.context = new EventMessageDataContext(storageAccount.TableEndpoint.AbsoluteUri, storageAccount.Credentials);
this.context.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1));
this.context.AddObject(this.context.EventMessageTableName, msg);
this.context.SaveChanges();
}
private void DeleteRecreateTable()
{
CloudTableClient client = AzureHelper.StorageAccount.CreateCloudTableClient();
client.RetryPolicy = RetryPolicies.Retry(int.MaxValue, TimeSpan.FromSeconds(5));
client.DeleteTable(WorkerStatsTableName);
client.CreateTable(WorkerStatsTableName);
}
/// <summary>
/// Instantiate a new DataSources object.
/// This method will throw exception if the storage account is wrong.
/// </summary>
public DataSources()
{
// Instantiate a table service context.
_context = new WordDataContext(_storageAccount.TableEndpoint.AbsoluteUri, _storageAccount.Credentials);
_context.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1));
// Instantiate a queue client for use.
_queueStorage = _storageAccount.CreateCloudQueueClient();
}
private void InitAzure()
{
this.StorageAccount = CloudStorageAccount.Parse(ConfigurationManager.ConnectionStrings["ComicStorage"].ConnectionString);
this.BlobClient = this.StorageAccount.CreateCloudBlobClient();
this.BlobClient.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.Zero);
this.QueueClient = this.StorageAccount.CreateCloudQueueClient();
this.QueueClient.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.Zero);
}
public AzureBlobContainer(CloudStorageAccount account, string containerName)
{
this.account = account;
var client = this.account.CreateCloudBlobClient();
client.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(5));
this.container = client.GetContainerReference(containerName);
}
/// <summary>
/// Gather data for each Hour based on Slice Start Time.
/// </summary>
/// <param name="sliceStartTime"></param>
/// <param name="urlFormat"></param>
private void GatherDataForOneHour(string sliceStartTime, string urlFormat)
{
Uri storageAccountUri = new Uri("http://" + _dataStorageAccountName + ".blob.core.windows.net/");
string year = sliceStartTime.Substring(0, 4);
string month = sliceStartTime.Substring(4, 2);
string day = sliceStartTime.Substring(6, 2);
string hour = sliceStartTime.Substring(8, 2);
string minute = sliceStartTime.Substring(10, 2);
DateTime dataSlotGathered = new DateTime(int.Parse(year), int.Parse(month), int.Parse(day), int.Parse(hour), int.Parse(minute), 0);
_logger.Write(TraceEventType.Information, "Current data slot gathered : {0}.......", dataSlotGathered);
// Temporary staging folder
string dataStagingFolder = string.Format(@"{0}\{1}\{1}-{2}", Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), year, month);
Directory.CreateDirectory(dataStagingFolder);
// Temporary staging file
string hourlyFileName = string.Format("data-{0}{1}{2}-{3}0000.txt", year, month, day, hour);
string decompressedFile = Path.Combine(dataStagingFolder, hourlyFileName);
try
{
_logger.Write(TraceEventType.Information, "Gathering hourly data: ..");
TriggerRequest(urlFormat, year, month, day, hour, decompressedFile);
_logger.Write(TraceEventType.Information, "Uploading to Blob: ..");
CloudBlobClient blobClient = new CloudBlobClient(storageAccountUri, new StorageCredentialsAccountAndKey(_dataStorageAccountName, _dataStorageAccountKey));
string blobPath = string.Format(CultureInfo.InvariantCulture, "httpdownloaddatain/{0}-{1}-{2}-{3}/{4}",
year, month, day, hour, hourlyFileName);
CloudBlobContainer container = blobClient.GetContainerReference(_dataStorageContainer);
container.CreateIfNotExist();
CloudBlob blob = container.GetBlobReference(blobPath);
BlobRequestOptions options = new BlobRequestOptions
{
Timeout = new TimeSpan(0, 5, 0),
RetryPolicy = RetryPolicies.Retry(3, new TimeSpan(0, 1, 0))
};
blob.UploadFile(decompressedFile, options);
}
catch (Exception ex)
{
_logger.Write(TraceEventType.Error, "Error occurred : {0}", ex);
}
finally
{
if (File.Exists(decompressedFile))
{
File.Delete(decompressedFile);
}
}
}
public BlobHelper(string storageName, string privateKey)
{
var connectionString = String.Format(
"DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageName, privateKey);
Account = CloudStorageAccount.Parse(connectionString);
BlobClient = Account.CreateCloudBlobClient();
BlobClient.RetryPolicy = RetryPolicies.Retry(4, TimeSpan.Zero);
}
public ExpenseExportStorage()
{
this.account = CloudConfiguration.GetStorageAccount(AzureConnectionStrings.DataConnection);
this.containerName = AzureStorageNames.ExpenseExportContainerName;
var client = this.account.CreateCloudBlobClient();
client.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(5));
this.container = client.GetContainerReference(this.containerName);
}
public GenericTableDataRepository(string connectionString)
{
this.connectionStringName = !string.IsNullOrEmpty(connectionString) ? connectionString : connectionStringName;
string cs = RoleEnvironment.GetConfigurationSettingValue(connectionStringName);
storageAccount = CloudStorageAccount.Parse(cs);
tableClient = new CloudTableClient(storageAccount.TableEndpoint.AbsoluteUri,
storageAccount.Credentials);
tableClient.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1));
tableClient.CreateTableIfNotExist(tableName);
}
public QueueHelper()
{
string connectionString = Settings.AzureConnectionString();
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(connectionString);
cloudQueueClient = cloudStorageAccount.CreateCloudQueueClient();
cloudQueueClient.Timeout = Settings.Timeout();
cloudQueueClient.RetryPolicy = RetryPolicies.Retry(Settings.RetryCount(), TimeSpan.FromSeconds(3));
}
public GuestBookService(string connectionString)
{
storageAccount = CloudStorageAccount.FromConfigurationSetting(connectionString);
string endpoint = storageAccount.TableEndpoint.AbsoluteUri;
var credentials = storageAccount.Credentials;
CloudTableClient.CreateTablesFromModel(typeof(GuestBookTableServiceContext), endpoint, credentials);
tableServiceContext = new GuestBookTableServiceContext(endpoint, credentials)
{
RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1))
};
}
private void InitBd()
{
Microsoft.WindowsAzure.CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
configSetter(RoleEnvironment.GetConfigurationSettingValue(configName));
});
storageAccount = CloudStorageAccount.FromConfigurationSetting("DataConnectionString");
CloudTableClient.CreateTablesFromModel(typeof(NewsDB_AzureDataContext), storageAccount.TableEndpoint.AbsoluteUri,
storageAccount.Credentials);
context = new NewsDB_AzureDataContext(storageAccount.TableEndpoint.AbsoluteUri, storageAccount.Credentials);
context.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(1));
}
private BlobRequestOptions GetBlobRequestOptions()
{
BlobRequestOptions options = new BlobRequestOptions()
{
// Changing for immediate return in case of disconnection
//RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultMaxBackoff),
RetryPolicy = RetryPolicies.Retry(0, TimeSpan.FromSeconds(0)), // i.e. do not retry
Timeout = TimeSpan.FromSeconds(AzureHelper.Timeout) // i.e. timeout in a second
};
return(options);
}
