public void Test()
{
var client = Play_all_for_BlobStreaming.GetCustom();
client.RetryPolicy = RetryPolicies.NoRetry();
var root = new BlobStreamingRoot(client);
var cont = root.GetContainer("tests").Create();
var storageItem = cont.GetItem("test");
storageItem.Write(w => w.WriteByte(1), options: StreamingWriteOptions.CompressIfPossible);
storageItem.ReadInto((props, stream) => stream.CopyTo(new MemoryStream(), 10));
var format = storageItem.GetInfo();
string ctx;
if (!format.Value.Properties.TryGetValue("ContentMD5", out ctx))
{
ctx = "None";
}
Console.WriteLine("MD5: {0}", ctx);
//storageItem.ReadText();
}
public AzureMessageLogWriter(CloudStorageAccount account, string tableName)
{
if (account == null)
{
throw new ArgumentNullException("account");
}
if (tableName == null)
{
throw new ArgumentNullException("tableName");
}
if (string.IsNullOrWhiteSpace(tableName))
{
throw new ArgumentException("tableName");
}
this.account = account;
this.tableName = tableName;
this.tableClient = account.CreateCloudTableClient();
this.tableClient.RetryPolicy = RetryPolicies.NoRetry();
var retryStrategy = new ExponentialBackoff(10, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(1));
this.retryPolicy = new RetryPolicy <StorageTransientErrorDetectionStrategy>(retryStrategy);
this.retryPolicy.ExecuteAction(() => tableClient.CreateTableIfNotExist(tableName));
}
static CloudBlobClient CreateCloudBlobClient()
{
var client = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudBlobClient();
client.RetryPolicy = RetryPolicies.NoRetry();
return(client);
}
private TableServiceContext CreateContext()
{
return(new TableServiceContext(this.account.TableEndpoint.ToString(), this.account.Credentials)
{
// retry policy is handled by TFHAB
RetryPolicy = RetryPolicies.NoRetry()
});
}
public AzureBlobContainer(CloudStorageAccount account, string containerName)
{
this.Account = account;
var client = account.CreateCloudBlobClient();
// retry policy is handled by TFHAB
client.RetryPolicy = RetryPolicies.NoRetry();
this.Container = client.GetContainerReference(containerName);
}
public AzureQueue(CloudStorageAccount account, string queueName, TimeSpan visibilityTimeout)
{
this.account = account;
this.visibilityTimeout = visibilityTimeout;
var client = this.account.CreateCloudQueueClient();
// retry policy is handled by TFHAB
client.RetryPolicy = RetryPolicies.NoRetry();
this.queue = client.GetQueueReference(queueName);
}
private List <Exception> SaveConcurrent(int numberOfCalls, SaveChangesOptions options)
{
// Do several calls to force some timeouts
// or manually disconnect the network while this test is running.
string partitionKey = Guid.NewGuid().ToString();
var exceptions = new ConcurrentBag <Exception>();
var entities = Enumerable.Range(0, numberOfCalls)
.Select(x => new TestServiceEntity {
PartitionKey = partitionKey, RowKey = x.ToString()
})
.ToList();
var barrier = new Barrier(numberOfCalls);
var countdown = new CountdownEvent(numberOfCalls);
foreach (var entity in entities)
{
ThreadPool.QueueUserWorkItem(x =>
{
try
{
var client = account.CreateCloudTableClient();
client.RetryPolicy = RetryPolicies.NoRetry();
// Explicitly set a VERY short timeout, to force a timeout very frequently.
client.Timeout = TimeSpan.FromSeconds(1);
var context = client.GetDataServiceContext();
context.AddObject(tableName, x);
barrier.SignalAndWait();
context.SaveChanges(options);
}
catch (Exception ex)
{
exceptions.Add(ex);
}
finally
{
countdown.Signal();
}
}, entity);
}
countdown.Wait();
if (exceptions.Count == 0)
{
Assert.Inconclusive("No exceptions were thrown to check if they are transient");
}
return(exceptions.ToList());
}
private TableServiceContext CreateContext()
{
var context = new TableServiceContext(this.account.TableEndpoint.ToString(), this.account.Credentials)
{
// retry policy is handled by TFHAB
RetryPolicy = RetryPolicies.NoRetry()
};
if (this.ReadWriteStrategy != null)
{
context.ReadingEntity += (sender, args) => this.ReadWriteStrategy.ReadEntity(context, args);
context.WritingEntity += (sender, args) => this.ReadWriteStrategy.WriteEntity(context, args);
}
return(context);
}
/// <summary>
/// Initializes a new instance of the reliable Windows Azure Table storage layer connected to the specified storage account,
/// initialized with the specified retry policy and custom serializer.
/// </summary>
/// <param name="storageAccountInfo">The access credentials for Windows Azure storage account.</param>
/// <param name="retryPolicy">The custom retry policy that will ensure reliable access to the underlying table storage.</param>
/// <param name="dataSerializer">An instance of the component which performs serialization and deserialization of storage objects.</param>
public ReliableCloudTableStorage(StorageAccountInfo storageAccountInfo, RetryPolicy retryPolicy, ICloudStorageEntitySerializer dataSerializer)
{
Guard.ArgumentNotNull(storageAccountInfo, "storageAccountInfo");
Guard.ArgumentNotNull(retryPolicy, "retryPolicy");
Guard.ArgumentNotNull(dataSerializer, "dataSerializer");
this.retryPolicy = retryPolicy;
this.dataSerializer = dataSerializer;
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(storageAccountInfo.AccountName, storageAccountInfo.AccountKey), true);
this.tableStorage = storageAccount.CreateCloudTableClient();
// Configure the Table storage not to enforce any retry policies since this is something that we will be dealing ourselves.
this.tableStorage.RetryPolicy = RetryPolicies.NoRetry();
}
public AzureBlobContainer(CloudStorageAccount account, string containerName)
{
this.Account = account;
var client = account.CreateCloudBlobClient();
// retry policy is handled by TFHAB
client.RetryPolicy = RetryPolicies.NoRetry();
this.Container = client.GetContainerReference(containerName);
this.writingStrategies = new Dictionary <Type, Action <IConcurrencyControlContext, T> >()
{
{ typeof(OptimisticConcurrencyContext), this.OptimisticControlContextWriteStrategy },
{ typeof(PessimisticConcurrencyContext), this.PessimisticControlContextWriteStrategy }
};
}
internal async Task MailSend(MailAddress mailAddress, string token)
{
PCSiteTraceSource.MethodStart();
try
{
var mailsender = new VerificationMailSender(new MailTemplateGateway(
new Blob(
Config.Item["Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString"],
Config.Item["Toyota.Gbook.WebSite.MailTemplate.ContainerName"],
RetryPolicies.NoRetry())));
await mailsender.SendPasswordReminderMail(mailAddress, token);
}
catch (Exception ex)
{
PCSiteTraceSource.AppError("メールGWを利用してのメール送信中に例外が発生しました。", ex);
throw;
}
}
public EventStore(CloudStorageAccount account, string tableName)
{
if (account == null)
{
throw new ArgumentNullException("account");
}
if (tableName == null)
{
throw new ArgumentNullException("tableName");
}
if (string.IsNullOrWhiteSpace(tableName))
{
throw new ArgumentException("tableName");
}
this.account = account;
this.tableName = tableName;
this.tableClient = account.CreateCloudTableClient();
this.tableClient.RetryPolicy = RetryPolicies.NoRetry();
// TODO: This could be injected.
var backgroundRetryStrategy = new ExponentialBackoff(10, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(1));
var blockingRetryStrategy = new Incremental(3, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(1));
this.pendingEventsQueueRetryPolicy = new RetryPolicy <StorageTransientErrorDetectionStrategy>(backgroundRetryStrategy);
this.pendingEventsQueueRetryPolicy.Retrying += (s, e) =>
{
var handler = this.Retrying;
if (handler != null)
{
handler(this, EventArgs.Empty);
}
Trace.TraceWarning("An error occurred in attempt number {1} to access table storage (PendingEventsQueue): {0}", e.LastException.Message, e.CurrentRetryCount);
};
this.eventStoreRetryPolicy = new RetryPolicy <StorageTransientErrorDetectionStrategy>(blockingRetryStrategy);
this.eventStoreRetryPolicy.Retrying += (s, e) => Trace.TraceWarning(
"An error occurred in attempt number {1} to access table storage (EventStore): {0}",
e.LastException.Message,
e.CurrentRetryCount);
this.eventStoreRetryPolicy.ExecuteAction(() => tableClient.CreateTableIfNotExist(tableName));
}
internal async Task MailSend(MailAddress mailAddress, List <string> internalMemberId)
{
PCSiteTraceSource.MethodStart();
try
{
var url = Config.Get <string>("Toyota.Gbook.WebSite.Mail.MailGW.Url");
var subject = Config.Get <string>("Toyota.Gbook.WebSite.IdRemind.Subject");
var mailsender = new VerificationMailSender(new MailTemplateGateway(
new Blob(
Config.Item["Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString"],
Config.Item["Toyota.Gbook.WebSite.MailTemplate.ContainerName"],
RetryPolicies.NoRetry())));
await mailsender.SendIdReminderMail(mailAddress, internalMemberId);
}
catch (Exception ex)
{
PCSiteTraceSource.AppError("メールGWを利用してのメール送信中に例外が発生しました。", ex);
}
}
public void then_not_found_error_is_not_transient()
{
// Fixes http://social.msdn.microsoft.com/forums/en-us/windowsazuredevelopment/thread/0fad898b-2fa2-4d8f-b918-3723194c2ef9
var client = account.CreateCloudBlobClient();
client.RetryPolicy = RetryPolicies.NoRetry();
try
{
string inexistantBlobName = Guid.NewGuid().ToString();
var blob = client.GetBlobReference(inexistantBlobName).DownloadByteArray();
Assert.Fail("Exception not thrown");
}
catch (StorageClientException ex)
{
Assert.IsInstanceOfType(ex.InnerException, typeof(WebException));
Assert.IsFalse(strategy.IsTransient(ex));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ReliableCloudBlobStorage"/> class using the specified storage account information, custom retry policy
/// and custom implementation of <see cref="ICloudStorageEntitySerializer"/> interface.
/// </summary>
/// <param name="storageAccountInfo">The access credentials for Windows Azure storage account.</param>
/// <param name="retryPolicy">The custom retry policy that will ensure reliable access to the underlying storage.</param>
/// <param name="dataSerializer">An instance of the component which performs serialization and deserialization of storage objects.</param>
public ReliableCloudBlobStorage(StorageAccountInfo storageAccountInfo, RetryPolicy retryPolicy, ICloudStorageEntitySerializer dataSerializer)
{
Guard.ArgumentNotNull(storageAccountInfo, "storageAccountInfo");
Guard.ArgumentNotNull(retryPolicy, "retryPolicy");
Guard.ArgumentNotNull(dataSerializer, "dataSerializer");
this.retryPolicy = retryPolicy;
this.dataSerializer = dataSerializer;
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(storageAccountInfo.AccountName, storageAccountInfo.AccountKey), true);
this.blobStorage = storageAccount.CreateCloudBlobClient();
// Configure the Blob storage not to enforce any retry policies since this is something that we will be dealing ourselves.
this.blobStorage.RetryPolicy = RetryPolicies.NoRetry();
// Disable parallelism in blob upload operations to reduce the impact of multiple concurrent threads on parallel upload feature.
this.blobStorage.ParallelOperationThreadCount = 1;
}
public CloudBlobStorage(CloudStorageAccount account, string rootContainerName)
{
this.account = account;
this.rootContainerName = rootContainerName;
blobClient = account.CreateCloudBlobClient();
blobClient.RetryPolicy = RetryPolicies.NoRetry();
readRetryPolicy = new RetryPolicy <StorageTransientErrorDetectionStrategy>(new Incremental(1, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(1)));
readRetryPolicy.Retrying += (s, e) => Trace.TraceWarning("An error occurred in attempt number {1} to read from blob storage: {0}", e.LastException.Message, e.CurrentRetryCount);
writeRetryPolicy = new RetryPolicy <StorageTransientErrorDetectionStrategy>(new FixedInterval(1, TimeSpan.FromSeconds(10))
{
FastFirstRetry = false
});
writeRetryPolicy.Retrying += (s, e) => Trace.TraceWarning("An error occurred in attempt number {1} to write to blob storage: {0}", e.LastException.Message, e.CurrentRetryCount);
var containerReference = blobClient.GetContainerReference(this.rootContainerName);
writeRetryPolicy.ExecuteAction(() => containerReference.CreateIfNotExist());
}
/// <summary>
/// Initializes a new instance of the <see cref="ReliableCloudQueueStorage"/> class using the specified storage account information, custom retry policy,
/// custom implementation of <see cref="ICloudStorageEntitySerializer"/> interface and custom implementation of the large message overflow store.
/// </summary>
/// <param name="storageAccountInfo">The storage account that is projected through this component.</param>
/// <param name="retryPolicy">The specific retry policy that will ensure reliable access to the underlying storage.</param>
/// <param name="dataSerializer">An instance of the component which performs serialization and deserialization of storage objects.</param>
/// <param name="overflowStorage">An instance of the component implementing overflow store that will be used for persisting large messages that cannot be accommodated in a queue due to message size constraints.</param>
public ReliableCloudQueueStorage(StorageAccountInfo storageAccountInfo, RetryPolicy retryPolicy, ICloudStorageEntitySerializer dataSerializer, ICloudBlobStorage overflowStorage)
{
Guard.ArgumentNotNull(storageAccountInfo, "storageAccountInfo");
Guard.ArgumentNotNull(retryPolicy, "retryPolicy");
Guard.ArgumentNotNull(dataSerializer, "dataSerializer");
Guard.ArgumentNotNull(overflowStorage, "overflowStorage");
this.retryPolicy = retryPolicy;
this.dataSerializer = dataSerializer;
this.overflowStorage = overflowStorage;
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(storageAccountInfo.AccountName, storageAccountInfo.AccountKey), true);
this.queueStorage = storageAccount.CreateCloudQueueClient();
// Configure the Queue storage not to enforce any retry policies since this is something that we will be dealing ourselves.
this.queueStorage.RetryPolicy = RetryPolicies.NoRetry();
this.inflightMessages = new ConcurrentDictionary <object, InflightMessageInfo>(Environment.ProcessorCount * 4, InflightMessageQueueInitialCapacity);
}
public AzureStorageConfigurationBuilder(CloudStorageAccount account)
{
// defaults
_queueConfig = client => client.RetryPolicy = RetryPolicies.NoRetry();
_blobConfig = client =>
{
client.RetryPolicy = RetryPolicies.NoRetry();
};
_tableConfig = client => client.RetryPolicy = RetryPolicies.NoRetry();
if (account.Credentials.AccountName == "devstoreaccount1")
{
_blobConfig += client =>
{
// http://stackoverflow.com/questions/4897826/
// local dev store works poorly with multi-thread uploads
client.ParallelOperationThreadCount = 1;
};
}
_account = account;
_accountId = account.Credentials.AccountName;
}
/// <summary>
/// Gets the number of blobs in the specified container.
/// </summary>
/// <param name="containerName">The name of the blob container to be queried.</param>
/// <returns>The number of blobs in the container.</returns>
public int GetCount(string containerName)
{
Guard.ArgumentNotNullOrEmptyString(containerName, "containerName");
var callToken = TraceManager.CloudStorageComponent.TraceIn(containerName);
int blobCount = 0;
try
{
var container = this.blobStorage.GetContainerReference(CloudUtility.GetSafeContainerName(containerName));
blobCount = this.retryPolicy.ExecuteAction <int>(() =>
{
var blobList = container.ListBlobs(new BlobRequestOptions()
{
BlobListingDetails = BlobListingDetails.None, RetryPolicy = RetryPolicies.NoRetry(), UseFlatBlobListing = true
});
return(blobList != null ? blobList.Count() : 0);
});
return(blobCount);
}
catch (StorageClientException ex)
{
// If blob doesn't exist, do not re-throw the exception, simply return a null reference.
if (!(ex.ErrorCode == StorageErrorCode.ContainerNotFound || ex.ErrorCode == StorageErrorCode.BlobNotFound || ex.ErrorCode == StorageErrorCode.ResourceNotFound))
{
throw;
}
return(0);
}
finally
{
TraceManager.CloudStorageComponent.TraceOut(callToken, blobCount);
}
}
public BlobHelper(CloudStorageAccount storageAccount)
{
_blobStorage = storageAccount.CreateCloudBlobClient();
_blobStorage.RetryPolicy = RetryPolicies.NoRetry();
}
