/// <summary>
/// The main entry point.
/// </summary>
/// <param name="args">
/// The command line arguments. Not used.
/// </param>
private static void Main(string[] args)
{
var credentials = new StorageCredentials("rwwa", "vyMfaSPxURHXZaIhhFJQRg5ZLEN6qDj4yU78r3oeOH+pZzdcf4S86QvGAsB6L8JaPti9qJbB929hy1Y9hipFmw==");
// Retrieve storage account from connection string.
var storageAccount = new CloudStorageAccount(credentials, true);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue
CloudQueue queue = queueClient.GetQueueReference("dataqueue");
queue.CreateIfNotExists();
Console.WriteLine("Up and listening to the queue.");
while (true)
{
CloudQueueMessage message;
do
{
message = queue.GetMessage();
if (message != null)
{
Console.WriteLine("Received Message for BLOB " + message.AsString);
var blobUrl = message.AsString;
var blockBlob = new CloudBlockBlob(new Uri(blobUrl), credentials);
if (blockBlob.Exists())
{
// TODO: download and process BLOB
/*using (var fileStream = System.IO.File.OpenWrite(@"path\myfile"))
{
blockBlob.DownloadToStream(fileStream);
}*/
blockBlob.Delete();
}
queue.DeleteMessage(message);
Console.WriteLine("BLOB " + message.AsString + " and queue message deleted.");
}
}
while (message != null);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
/// <summary>
/// The main entry point.
/// </summary>
/// <param name="args">
/// The command line arguments. Not used.
/// </param>
private static void Main(string[] args)
{
var credentials = new StorageCredentials("[Enter your account name]", "[Enter your account key]");
// Retrieve storage account from connection string.
var storageAccount = new CloudStorageAccount(credentials, true);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue
CloudQueue queue = queueClient.GetQueueReference("dataqueue");
Console.WriteLine("Up and listening to the queue.");
while (true)
{
CloudQueueMessage message;
do
{
message = queue.GetMessage();
if (message != null)
{
Console.WriteLine("Received Message for BLOB " + message.AsString);
var blobUrl = message.AsString;
var blockBlob = new CloudBlockBlob(new Uri(blobUrl), credentials);
if (blockBlob.Exists())
{
// TODO: download and process BLOB
/*using (var fileStream = System.IO.File.OpenWrite(@"path\myfile"))
{
blockBlob.DownloadToStream(fileStream);
}*/
blockBlob.Delete();
}
queue.DeleteMessage(message);
Console.WriteLine("BLOB " + message.AsString + " and queue message deleted.");
}
}
while (message != null);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
public override void Run()
{
storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
cmdQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlercmd");
urlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlerurl");
errorQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlererror");
lastTenUrlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("lasttenurlcrawled");
index = storageAccount.CreateCloudTableClient().GetTableReference("krawlerindex");
visisted = new HashSet<string>();
cmdQueue.CreateIfNotExists();
urlQueue.CreateIfNotExists();
errorQueue.CreateIfNotExists();
lastTenUrlQueue.CreateIfNotExists();
// This is a sample worker implementation. Replace with your logic.
Trace.TraceInformation("WorkerRole entry point called", "Information");
Crawler crawler = new Crawler();
//cmdQueue.Clear();
//cmdQueue.AddMessage(new CloudQueueMessage("stop"));
while (true)
{
Thread.Sleep(500);
Trace.TraceInformation("Working", "Information");
CloudQueueMessage cmd = cmdQueue.PeekMessage();
if (cmd == null || cmd.AsString == "stop")
{
continue;
}
else if (cmd.AsString == "start")
{
CloudQueueMessage url = urlQueue.GetMessage();
if (!visisted.Contains(url.AsString))
{
crawler.Crawl(url.AsString);
visisted.Add(url.AsString);
}
urlQueue.DeleteMessage(url);
}
else if (cmd.AsString == "load" && urlQueue.PeekMessage() == null)
{
crawler.LoadQueue("http://www.cnn.com");
}
else
{
continue;
}
}
}
public Crawler()
{
storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
urlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlerurl");
cmdQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlercmd");
index = storageAccount.CreateCloudTableClient().GetTableReference("krawlerindex");
errorQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlererror");
lastTenUrlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("lasttenurlcrawled");
crawlStatTable = storageAccount.CreateCloudTableClient().GetTableReference("krawlerstattable");
disallowedPaths = new List<string>();
visited = new HashSet<string>();
crawlStatTable.CreateIfNotExists();
crawlCounter = (NumberOfUrlCrawled() != 0) ? NumberOfUrlCrawled() : 0;
}
private static CloudQueue MakeQueue(CloudStorageAccount account)
{
var client = account.CreateCloudQueueClient();
client.RetryPolicy = new ExponentialRetry(new TimeSpan(0, 0, 0, 2), 10);
return client.GetQueueReference(QueueName);
}
public static void Run(string connectionString, bool disableLogging)
{
_connectionString = connectionString;
_storageAccount = CloudStorageAccount.Parse(connectionString);
_queueClient = _storageAccount.CreateCloudQueueClient();
try
{
TimeSpan azureSDKTime = RunAzureSDKTest();
TimeSpan webJobsSDKTime = RunWebJobsSDKTest(disableLogging);
// Convert to ulong because the measurment block does not support other data type
ulong perfRatio = (ulong)((webJobsSDKTime.TotalMilliseconds / azureSDKTime.TotalMilliseconds) * 100);
Console.WriteLine("--- Results ---");
Console.WriteLine("Azure SDK: {0} ms: ", azureSDKTime.TotalMilliseconds);
Console.WriteLine("WebJobs SDK: {0} ms: ", webJobsSDKTime.TotalMilliseconds);
Console.WriteLine("Perf ratio (x100, long): {0}", perfRatio);
MeasurementBlock.Mark(
perfRatio,
(disableLogging ? QueueNoLoggingOverheadMetric : QueueLoggingOverheadMetric) + ";Ratio;Percent");
}
finally
{
Cleanup();
}
}
/// <summary>
/// Get the service properties
/// </summary>
/// <param name="account">Cloud storage account</param>
/// <param name="type">Service type</param>
/// <param name="options">Request options</param>
/// <param name="operationContext">Operation context</param>
/// <returns>The service properties of the specified service type</returns>
public XSCLProtocol.ServiceProperties GetStorageServiceProperties(StorageServiceType type, IRequestOptions options, XSCL.OperationContext operationContext)
{
XSCL.CloudStorageAccount account = StorageContext.StorageAccount;
try
{
switch (type)
{
case StorageServiceType.Blob:
return(account.CreateCloudBlobClient().GetServicePropertiesAsync((BlobRequestOptions)options, operationContext).Result);
case StorageServiceType.Queue:
return(account.CreateCloudQueueClient().GetServicePropertiesAsync((QueueRequestOptions)options, operationContext).Result);
case StorageServiceType.File:
FileServiceProperties fileServiceProperties = account.CreateCloudFileClient().GetServicePropertiesAsync((FileRequestOptions)options, operationContext).Result;
XSCLProtocol.ServiceProperties sp = new XSCLProtocol.ServiceProperties();
sp.Clean();
sp.Cors = fileServiceProperties.Cors;
sp.HourMetrics = fileServiceProperties.HourMetrics;
sp.MinuteMetrics = fileServiceProperties.MinuteMetrics;
return(sp);
default:
throw new ArgumentException(Resources.InvalidStorageServiceType, "type");
}
}
catch (AggregateException e) when(e.InnerException is XSCL.StorageException)
{
throw e.InnerException;
}
}
protected virtual void configureQueue(CloudStorageAccount storageAccount)
{
// Create the queue client
this.queueClient = storageAccount.CreateCloudQueueClient();
this.queue = createQueue(this.queueClient, this.queueName);
}
public BlobManager(string conStr)
{
//RoleEnvironment.GetConfigurationSettingValue("UploadCon")
Storage = CloudStorageAccount.Parse(conStr);
BlobClient = Storage.CreateCloudBlobClient();
QueueClient = Storage.CreateCloudQueueClient();
}
public static void Initialize(string storageConnectionString, string ocrQueueName, string emailQueueName)
{
storageAccount = CloudStorageAccount.Parse(storageConnectionString);
queueClient = storageAccount.CreateCloudQueueClient();
ocrQueue = InitializeQueue(ocrQueueName);
emailQueue = InitializeQueue(emailQueueName);
}
// Constructor - pass in a storage connection string.
public QueueHelper(string connectionString)
{
Account = CloudStorageAccount.Parse(connectionString);
QueueClient = Account.CreateCloudQueueClient();
IRetryPolicy linearRetryPolicy = new LinearRetry(TimeSpan.Zero, 4);
QueueClient.RetryPolicy = linearRetryPolicy;
}
/// <summary>
/// Constructs the transport
/// </summary>
public AzureStorageQueuesTransport(CloudStorageAccount storageAccount, string inputQueueName)
{
if (storageAccount == null) throw new ArgumentNullException("storageAccount");
if (inputQueueName == null) throw new ArgumentNullException("inputQueueName");
_inputQueueName = inputQueueName.ToLowerInvariant();
_queueClient = storageAccount.CreateCloudQueueClient();
}
static StorageHelper()
{
if (!string.IsNullOrEmpty(SimpleSettings.StorageConnectionString))
{
StorageAccount = CloudStorageAccount.Parse(SimpleSettings.StorageConnectionString);
BlobClient = StorageAccount.CreateCloudBlobClient();
QueueClient = StorageAccount.CreateCloudQueueClient();
}
}
public CloudQueue GetQueueTable(string tableName)
{
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse("UseDevelopmentStorage=true");
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference(tableName);
queue.CreateIfNotExistsAsync();
return(queue);
}
private void CreateQueues(CloudStorageAccount storageAccount)
{
var mailQueueName = CloudConfigurationManager.GetSetting("MailQueue.Name");
var queueClient = storageAccount.CreateCloudQueueClient();
var queue = queueClient.GetQueueReference(mailQueueName);
queue.CreateIfNotExists();
}
public QueueRepository()
{
try
{
_storageAccount = CloudStorageAccount.Parse(ConfigReader<string>.GetSetting("EarthOnlineStorage"));
_blobClient = _storageAccount.CreateCloudBlobClient();
_queueClient = _storageAccount.CreateCloudQueueClient();
}
catch (Exception) { }//fail silently for developers without valid storage settings
}
public azureStorageHelper(string acountName, string accountKey)
{
sCredentials = new StorageCredentials(acountName, accountKey);
storageAccount = new CloudStorageAccount(sCredentials, true);
queueClient = storageAccount.CreateCloudQueueClient();
blobClient = storageAccount.CreateCloudBlobClient();
permisions = new BlobContainerPermissions();
permisions.PublicAccess = BlobContainerPublicAccessType.Container;
}
public static async Task Init(string azureStorageConnectionString)
{
storageAccount = CloudStorageAccount.Parse(azureStorageConnectionString);
storageAccountForQueue = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(azureStorageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
var queueClient = storageAccountForQueue.CreateCloudQueueClient();
// Get reference for storage tables and queues
var countryCloudTable = tableClient.GetTableReference("Country");
var vstsInstanceCloudTable = tableClient.GetTableReference("VstsInstance");
var vstsUserCloudTable = tableClient.GetTableReference("VstsUser");
var talentChallengeCloudTable = tableClient.GetTableReference("TalentChallenges");
var vstsavailableinstancesWestEurope = queueClient.GetQueueReference($"vstsavailableinstances-westeurope");
var vstspendingavailableinstancesQueue = queueClient.GetQueueReference($"vstspendingavailableinstances");
// Create the storage tables and queues if not exists
await vstsavailableinstancesWestEurope.CreateIfNotExistsAsync();
await vstspendingavailableinstancesQueue.CreateIfNotExistsAsync();
var isCountryInitialized = await countryCloudTable.ExistsAsync();
var isVSTSInstanceInitialized = await vstsInstanceCloudTable.ExistsAsync();
var isVSTSUserInitialized = await vstsUserCloudTable.ExistsAsync();
var isTalentChallengeInitialized = await talentChallengeCloudTable.ExistsAsync();
// Init data
if (!isCountryInitialized)
{
await countryCloudTable.CreateAsync();
await InitCountries(countryCloudTable);
}
if (!isVSTSInstanceInitialized)
{
await vstsInstanceCloudTable.CreateAsync();
await InitVSTSInstances(vstsInstanceCloudTable, vstspendingavailableinstancesQueue);
}
if (!isVSTSUserInitialized)
{
await vstsUserCloudTable.CreateAsync();
}
if (!isTalentChallengeInitialized)
{
await talentChallengeCloudTable.CreateAsync();
}
}
public PickCreator()
{
_storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
_queueClient = _storageAccount.CreateCloudQueueClient();
_queue = _queueClient.GetQueueReference("scaleoutsamplequeue");
_queue.CreateIfNotExists();
}
public override bool OnStart()
{
_account = CloudStorageAccount.Parse(
RoleEnvironment.GetConfigurationSettingValue("SiteMonitRConnectionString")
);
_queueClient = _account.CreateCloudQueueClient();
_queue = _queueClient.GetQueueReference(new WebSiteQueueConfiguration().GetIncomingQueueName());
_queue.CreateIfNotExists();
return base.OnStart();
}
public AzureQueueHelper()
{
// Retrieve the storage account from a connection string in the web.config file.
storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["AzureQueueConnectionString"]);
// Create the cloud queue client.
queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to our queue.
queue = queueClient.GetQueueReference("receiptgenerator");
}
/// <summary>
/// Constructs the transport
/// </summary>
public AzureStorageQueuesTransport(CloudStorageAccount storageAccount, string inputQueueName, IRebusLoggerFactory rebusLoggerFactory)
{
if (storageAccount == null) throw new ArgumentNullException(nameof(storageAccount));
_queueClient = storageAccount.CreateCloudQueueClient();
_log = rebusLoggerFactory.GetCurrentClassLogger();
if (inputQueueName != null)
{
_inputQueueName = inputQueueName.ToLowerInvariant();
}
}
protected AzureStorageQueueEndpointManager(AzureStorageQueueEndpoint endpoint,
IAzureStorageConfiguration storageConfiguration)
{
if (storageConfiguration == null)
throw new ArgumentNullException("storageConfiguration");
endpoint.Validate();
storageConfiguration.Validate();
CloudStorageAccount = CloudStorageAccount.Parse(storageConfiguration.ConnectionString);
CloudQueueClient = CloudStorageAccount.CreateCloudQueueClient();
CloudQueue = CloudQueueClient.GetQueueReference(endpoint.QueueName);
}
static void PrepareAccount(CloudStorageAccount account)
{
var table = account.CreateCloudTableClient();
var tableRef = table.GetTableReference(TableName);
tableRef.CreateIfNotExists();
var blob = account.CreateCloudBlobClient();
var blobRef = blob.GetContainerReference(ContainerName);
blobRef.CreateIfNotExists();
var queue = account.CreateCloudQueueClient();
var queueRef = queue.GetQueueReference(QueueName);
queueRef.CreateIfNotExists();
}
public static void Initialize(string account, string key)
{
Uri blobUri = new Uri(string.Format("http://{0}.blob.core.windows.net", account));
Uri queueUri = new Uri(string.Format("http://{0}.queue.core.windows.net", account));
Uri tableUri = new Uri(string.Format("http://{0}.table.core.windows.net", account));
s_credentials = new StorageCredentials(account, key);
//s_storageAccount = new CloudStorageAccount(s_credentials, blobUri, queueUri, tableUri);
s_storageAccount = new CloudStorageAccount(s_credentials, true);
s_blobClient = s_storageAccount.CreateCloudBlobClient();
s_tableClient = s_storageAccount.CreateCloudTableClient();
s_queueClient = s_storageAccount.CreateCloudQueueClient();
}
public static QueueStorage Create(CloudStorageAccount storageAccount, string queueReference )
{
try
{
var queueStorage = storageAccount.CreateCloudQueueClient();
CloudQueue queue = queueStorage.GetQueueReference(queueReference);
queue.CreateIfNotExists();
return new QueueStorage(queueStorage, queue);
}
catch (Exception exception)
{
throw new CloudStorageClientException(exception);
}
}
/// <summary>
/// Initializes a new instance of ConnectionSettings using the given CloudStorageAccount and queue name.
/// </summary>
/// <param name="storageAccount">
/// The CloudStorageAccount to use to connect to the Azure Storage Account.
/// </param>
/// <param name="queueName">
/// The name of the queue to connect to.
/// </param>
public ConnectionSettings(CloudStorageAccount storageAccount, string queueName)
{
if (null == storageAccount)
{
throw new ArgumentNullException("storageAccount");
}
else if (String.IsNullOrWhiteSpace(queueName))
{
throw new ArgumentNullException("queueName");
}
client = storageAccount.CreateCloudQueueClient();
QueueName = queueName;
}
public JobQueue()
{
// var connectionString = RoleEnvironment.GetConfigurationSettingValue("PhotoAppStorage");
//var connectionString = ConfigurationManager.ConnectionStrings["PhotoAppStorage"].ConnectionString;
//var account = CloudStorageAccount.Parse(connectionString);
//var account = CloudStorageAccount.DevelopmentStorageAccount;
var cred = new StorageCredentials("jholm",
"/bVipQ2JxjWwYrZQfHmzhaBx1p1s8BoD/wX6VWOmg4/gpVo/aALrjsDUKqzXsFtc9utepPqe65NposrXt9YsyA==");
var account = new CloudStorageAccount(cred, true);
var cloudQueueClient = account.CreateCloudQueueClient();
_cloudQueue = cloudQueueClient.GetQueueReference(_queueName);
_cloudQueue.CreateIfNotExists();
}
public Microsoft.WindowsAzure.Storage.Queue.CloudQueue GetStorageQueue(string name)
{
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(
StorageConnectionString);
ServicePoint tableServicePoint = ServicePointManager.FindServicePoint(storageAccount.TableEndpoint);
tableServicePoint.UseNagleAlgorithm = false;
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient client = storageAccount.CreateCloudQueueClient();
CloudQueue queue = client.GetQueueReference(name);
queue.CreateIfNotExistsAsync().Wait();
return(queue);
}
public void Initialise(string queueName)
{
if (string.IsNullOrEmpty(queueName))
{
throw new ArgumentException("QueueName cannot be null or empty", "queueName");
}
string connStr = ConfigManager.ConfigManager.GetConfigurationSetting(ConfigManager.ConfigManager.DefaultQueueStorageConnString, string.Empty);
_storageAccount = CloudStorageAccount.Parse(connStr);
_queueStorage = _storageAccount.CreateCloudQueueClient();
_queue = _queueStorage.GetQueueReference(queueName);
_queue.CreateIfNotExists();
}
public QueueTransport(CloudStorageAccount account, string prefix, string inputQueueName)
{
this.account = account;
this.client = account.CreateCloudQueueClient();
this.queues = new ConcurrentDictionary<string, Lazy<Task<CloudQueue>>>();
if (prefix == null)
{
this.queueNameFormat = "{0}";
}
else
{
this.queueNameFormat = string.Format("{0}-{{0}}", prefix);
}
this.inputQueue = new Lazy<Task<CloudQueue>>(() => GetOrCreateQueue(inputQueueName));
}
public async Task <VstsInstanceEntity> GetAvailableInstance(string azureLocation)
{
CloudQueueClient queueClient = storageAccountForQueue.CreateCloudQueueClient();
// Retrieve a reference to a container.
CloudQueue queue = queueClient.GetQueueReference($"vstsavailableinstances-{azureLocation}");
// Get the next message
CloudQueueMessage availableInstance = await queue.GetMessageAsync();
if (availableInstance == null)
{
throw new Exception($"No more availble VSTS instance on region [{azureLocation}]. Please try later.");
}
var instanceName = availableInstance.AsString;
var vstsInstanceEntity = await Get(instanceName);
await queue.DeleteMessageAsync(availableInstance);
return(vstsInstanceEntity);
}
/// <summary>
/// The work.
/// </summary>
public void Work()
{
var credentials = new StorageCredentials(Constants.StorageAccountName, Constants.StorageAccountKey);
var storageAccount = new CloudStorageAccount(credentials, true);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue schedulerQueue = queueClient.GetQueueReference(Constants.SchedulerQueue);
schedulerQueue.CreateIfNotExists();
var fileQueue = queueClient.GetQueueReference(Constants.FileQueue);
fileQueue.CreateIfNotExists();
while (!this.isCancel)
{
CloudQueueMessage schedulerMessage;
do
{
schedulerMessage = schedulerQueue.GetMessage();
if (schedulerMessage != null)
{
Console.WriteLine("Received indexing message. Start Indexing...");
var directory = new DirectoryInfo(Constants.LocalDirectory);
foreach (var file in directory.EnumerateFiles())
{
var fileMessage = new CloudQueueMessage(file.FullName);
fileQueue.AddMessage(fileMessage);
}
Console.WriteLine("File queue filled.");
schedulerQueue.DeleteMessage(schedulerMessage);
}
}
while (schedulerMessage != null);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
public CloudContext(Cache cacheService)
{
_storageAccount = CloudStorageAccount.Parse(ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString);
_tableClient = _storageAccount.CreateCloudTableClient();
_tableClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromMilliseconds(2000), 3);
_tableTicktes = _tableClient.GetTableReference("TicketsRead");
_tableEvents = _tableClient.GetTableReference("EventsRead");
_tableMyEvents = _tableClient.GetTableReference("MyEventsRead");
_tableTicktes.CreateIfNotExists();
_tableEvents.CreateIfNotExists();
_tableMyEvents.CreateIfNotExists();
_queueClient = _storageAccount.CreateCloudQueueClient();
_queue = _queueClient.GetQueueReference("ticket-online");
_queue.CreateIfNotExists();
_cache = cacheService;
}
public async Task RunQueueTest(SharedAccessAccountPolicy policy, int?httpsPort, OperationContext opContext = null)
{
CloudQueueClient queueClient = GenerateCloudQueueClient();
string queueName = "q" + Guid.NewGuid().ToString("N");
try
{
CloudStorageAccount account = new CloudStorageAccount(queueClient.Credentials, false);
string accountSASToken = account.GetSharedAccessSignature(policy);
StorageCredentials accountSAS = new StorageCredentials(accountSASToken);
StorageUri storageUri = queueClient.StorageUri;
if (httpsPort != null)
{
storageUri = new StorageUri(TransformSchemeAndPort(storageUri.PrimaryUri, "https", httpsPort.Value), TransformSchemeAndPort(storageUri.SecondaryUri, "https", httpsPort.Value));
}
else
{
storageUri = new StorageUri(TransformSchemeAndPort(storageUri.PrimaryUri, "http", 80), TransformSchemeAndPort(storageUri.SecondaryUri, "http", 80));
}
CloudStorageAccount accountWithSAS = new CloudStorageAccount(accountSAS, null, storageUri, null, null);
CloudQueueClient queueClientWithSAS = accountWithSAS.CreateCloudQueueClient();
CloudQueue queueWithSAS = queueClientWithSAS.GetQueueReference(queueName);
CloudQueue queue = queueClient.GetQueueReference(queueName);
await queue.CreateAsync();
string messageText = "message text";
CloudQueueMessage message = new CloudQueueMessage(messageText);
await queue.AddMessageAsync(message);
Assert.AreEqual(messageText, (await queueWithSAS.GetMessageAsync(null, null, opContext)).AsString);
}
finally
{
queueClient.GetQueueReference(queueName).DeleteIfExistsAsync().Wait();
}
}
/// <summary>
/// The work.
/// </summary>
public void Work()
{
var credentials = new StorageCredentials(Constants.StorageAccountName, Constants.StorageAccountKey);
var storageAccount = new CloudStorageAccount(credentials, true);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue fileQueue = queueClient.GetQueueReference(Constants.FileQueue);
fileQueue.CreateIfNotExists();
while (!this.isCancel)
{
CloudQueueMessage message;
do
{
message = fileQueue.GetMessage();
if (message != null)
{
var file = new FileInfo(message.AsString);
if (file.Exists)
{
// TODO: Process file...
file.Delete();
Console.WriteLine("File " + file.Name + " deleted.");
}
else
{
Console.WriteLine("ERROR: Received message for non-existing file");
}
fileQueue.DeleteMessage(message);
}
}
while (message != null);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
/// <summary>
/// Set service properties
/// </summary>
/// <param name="account">Cloud storage account</param>
/// <param name="type">Service type</param>
/// <param name="properties">Service properties</param>
/// <param name="options">Request options</param>
/// <param name="operationContext">Operation context</param>
public void SetStorageServiceProperties(StorageServiceType type, XSCLProtocol.ServiceProperties properties, IRequestOptions options, XSCL.OperationContext operationContext)
{
XSCL.CloudStorageAccount account = StorageContext.StorageAccount;
try
{
switch (type)
{
case StorageServiceType.Blob:
Task.Run(() => account.CreateCloudBlobClient().SetServicePropertiesAsync(properties, (BlobRequestOptions)options, operationContext)).Wait();
break;
case StorageServiceType.Queue:
Task.Run(() => account.CreateCloudQueueClient().SetServicePropertiesAsync(properties, (QueueRequestOptions)options, operationContext)).Wait();
break;
case StorageServiceType.File:
if (null != properties.Logging)
{
throw new InvalidOperationException(Resources.FileNotSupportLogging);
}
FileServiceProperties fileServiceProperties = new FileServiceProperties();
fileServiceProperties.Cors = properties.Cors;
fileServiceProperties.HourMetrics = properties.HourMetrics;
fileServiceProperties.MinuteMetrics = properties.MinuteMetrics;
Task.Run(() => account.CreateCloudFileClient().SetServicePropertiesAsync(fileServiceProperties, (FileRequestOptions)options, operationContext)).Wait();
break;
default:
throw new ArgumentException(Resources.InvalidStorageServiceType, "type");
}
}
catch (AggregateException e) when(e.InnerException is XSCL.StorageException)
{
throw e.InnerException;
}
}
public async Task RunPermissionsTestQueues(SharedAccessAccountPolicy policy)
{
CloudQueueClient queueClient = GenerateCloudQueueClient();
string queueName = "q" + Guid.NewGuid().ToString("N");
try
{
CloudStorageAccount account = new CloudStorageAccount(queueClient.Credentials, false);
string accountSASToken = account.GetSharedAccessSignature(policy);
StorageCredentials accountSAS = new StorageCredentials(accountSASToken);
CloudStorageAccount accountWithSAS = new CloudStorageAccount(accountSAS, null, queueClient.StorageUri, null, null);
CloudQueueClient queueClientWithSAS = accountWithSAS.CreateCloudQueueClient();
CloudQueue queueWithSAS = queueClientWithSAS.GetQueueReference(queueName);
CloudQueue queue = queueClient.GetQueueReference(queueName);
// General pattern - If current perms support doing a thing with SAS, do the thing with SAS and validate with shared
// Otherwise, make sure SAS fails and then do the thing with shared key.
// Things to do:
// Create the queue (Create or Write perms, Container RT)
// List queues (List perms, Service RT)
// Set queue metadata (Write perms, Container RT)
// Insert a message (Add perms, Object RT)
// Peek a message (Read perms, Object RT)
// Get a message (Process perms, Object RT)
// Update a message (Update perms, Object RT)
// Clear all messages (Delete perms, Object RT)
if ((((policy.Permissions & SharedAccessAccountPermissions.Create) == SharedAccessAccountPermissions.Create) || ((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write)) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Container) == SharedAccessAccountResourceTypes.Container))
{
await queueWithSAS.CreateAsync();
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await queueWithSAS.CreateAsync(), "Creating a queue with SAS should fail without Add and Container-level permissions.");
await queue.CreateAsync();
}
Assert.IsTrue(await queue.ExistsAsync());
if (((policy.Permissions & SharedAccessAccountPermissions.List) == SharedAccessAccountPermissions.List) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Service) == SharedAccessAccountResourceTypes.Service))
{
Assert.AreEqual(queueName, (await queueClientWithSAS.ListQueuesSegmentedAsync(queueName, null)).Results.First().Name);
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => (await queueClientWithSAS.ListQueuesSegmentedAsync(queueName, null)).Results.First(), "Listing queues with SAS should fail without Read and Service-level permissions.");
}
queueWithSAS.Metadata["metadatakey"] = "metadatavalue";
if (((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Container) == SharedAccessAccountResourceTypes.Container))
{
await queueWithSAS.SetMetadataAsync();
await queue.FetchAttributesAsync();
Assert.AreEqual("metadatavalue", queue.Metadata["metadatakey"]);
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await queueWithSAS.SetMetadataAsync(), "Setting a queue's metadata with SAS should fail without Write and Container-level permissions.");
}
string messageText = "messageText";
CloudQueueMessage message = new CloudQueueMessage(messageText);
if (((policy.Permissions & SharedAccessAccountPermissions.Add) == SharedAccessAccountPermissions.Add) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
await queueWithSAS.AddMessageAsync(message);
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await queueWithSAS.AddMessageAsync(message), "Adding a queue message should fail with SAS without Add and Object-level permissions.");
await queue.AddMessageAsync(message);
}
Assert.AreEqual(messageText, ((await queue.PeekMessageAsync()).AsString));
if (((policy.Permissions & SharedAccessAccountPermissions.Read) == SharedAccessAccountPermissions.Read) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
Assert.AreEqual(messageText, (await queueWithSAS.PeekMessageAsync()).AsString);
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await queueWithSAS.PeekMessageAsync(), "Peeking a queue message should fail with SAS without Read and Object-level permissions.");
}
CloudQueueMessage messageResult = null;
if (((policy.Permissions & SharedAccessAccountPermissions.ProcessMessages) == SharedAccessAccountPermissions.ProcessMessages) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
messageResult = await queueWithSAS.GetMessageAsync();
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await queueWithSAS.GetMessageAsync(), "Getting a message should fail with SAS without Process and Object-level permissions.");
messageResult = await queue.GetMessageAsync();
}
Assert.AreEqual(messageText, messageResult.AsString);
string newMessageContent = "new content";
messageResult.SetMessageContent(newMessageContent);
if (((policy.Permissions & SharedAccessAccountPermissions.Update) == SharedAccessAccountPermissions.Update) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
await queueWithSAS.UpdateMessageAsync(messageResult, TimeSpan.Zero, MessageUpdateFields.Content | MessageUpdateFields.Visibility);
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await queueWithSAS.UpdateMessageAsync(messageResult, TimeSpan.Zero, MessageUpdateFields.Content | MessageUpdateFields.Visibility), "Updating a message should fail with SAS without Update and Object-level permissions.");
await queue.UpdateMessageAsync(messageResult, TimeSpan.Zero, MessageUpdateFields.Content | MessageUpdateFields.Visibility);
}
messageResult = await queue.PeekMessageAsync();
Assert.AreEqual(newMessageContent, messageResult.AsString);
if (((policy.Permissions & SharedAccessAccountPermissions.Delete) == SharedAccessAccountPermissions.Delete) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
await queueWithSAS.ClearAsync();
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await queueWithSAS.ClearAsync(), "Clearing messages should fail with SAS without delete and Object-level permissions.");
}
}
finally
{
queueClient.GetQueueReference(queueName).DeleteIfExistsAsync().Wait();
}
}