public void Setup() { client = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudQueueClient(); client.ServerTimeout = TimeSpan.FromSeconds(10); nativeQueue = client.GetQueueReference(QueueName); nativeQueue.CreateIfNotExists(); nativeQueue.Clear(); sender = new AzureMessageQueueSender { Client = client, MessageSerializer = new JsonMessageSerializer(new MessageMapper()) }; sender.Init(QueueName, true); receiver = new AzureMessageQueueReceiver { Client = client, MessageSerializer = new JsonMessageSerializer(new MessageMapper()), }; receiver.Init(QueueName, true); }
private void InitQueue() { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); queue = queueClient.GetQueueReference("jobqueue"); queue.CreateIfNotExists(); }
public Crawler() { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); table = tableClient.GetTableReference("crawlertable"); table.CreateIfNotExists(); datatable = tableClient.GetTableReference("datatable"); datatable.CreateIfNotExists(); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); urlQueue = queueClient.GetQueueReference("urlqueue"); urlQueue.CreateIfNotExists(); adminQueue = queueClient.GetQueueReference("adminqueue"); adminQueue.CreateIfNotExists(); alreadyVisitedUrls = new HashSet<String>(); disallowedUrls = new HashSet<String>(); errorUrls = new HashSet<String>(); tableSize = 0; totalUrls = 0; counter = 1; compareDate = DateTime.ParseExact("2015-04-01", "yyyy-MM-dd", CultureInfo.InvariantCulture); //Regex to check for valid html document rgx = new Regex(@"^[a-zA-Z0-9\-]+.?(htm|html)?$"); }
public override bool OnStart() { // Set the maximum number of concurrent connections. ServicePointManager.DefaultConnectionLimit = 12; // Read database connection string and open database. var dbConnString = CloudConfigurationManager.GetSetting("TCWAdminPortalDbConnectionString"); _dbContext = new TCWAdminContext(dbConnString); // Open storage account using credentials from .cscfg file. var storageAccount = CloudStorageAccount.Parse (RoleEnvironment.GetConfigurationSettingValue("StorageConnectionString")); Trace.TraceInformation("Creating images blob container"); var blobClient = storageAccount.CreateCloudBlobClient(); _imagesBlobContainer = blobClient.GetContainerReference("images"); if (_imagesBlobContainer.CreateIfNotExists()) { // Enable public access on the newly created "images" container. _imagesBlobContainer.SetPermissions( new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }); } Trace.TraceInformation("Creating images queue"); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); _imagesQueue = queueClient.GetQueueReference("images"); _imagesQueue.CreateIfNotExists(); Trace.TraceInformation("Storage initialized"); return base.OnStart(); }
public override bool OnStart() { ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount; ConfigureDiagnostics(); // Read storage account configuration settings Trace.TraceInformation("Initializing storage account in WorkerC"); var storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue( "StorageConnectionString" )); // Initialize queue storage Trace.TraceInformation("Creating queue client in WorkerC"); var queueClient = storageAccount.CreateCloudQueueClient(); subscribeQueue = queueClient.GetQueueReference("subscribequeue"); // Initialize table storage Trace.TraceInformation("Creating table client in WorkerC"); var tableClient = storageAccount.CreateCloudTableClient(); subscribersTable = tableClient.GetTableReference("Subscribers"); Trace.TraceInformation("WorkerC: Creating blob container, queue, tables, if they don't exist."); subscribeQueue.CreateIfNotExists(); subscribersTable.CreateIfNotExists(); return base.OnStart(); }
public override bool OnStart() { ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount; // Read storage account configuration settings ConfigureDiagnostics(); Trace.TraceInformation("Initializing storage account in worker role B"); var storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("StorageConnectionString")); // Initialize queue storage Trace.TraceInformation("Creating queue client."); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); sendEmailQueue = queueClient.GetQueueReference("azuremailqueue"); subscribeQueue = queueClient.GetQueueReference("azuremailsubscribequeue"); // Initialize blob storage CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); blobContainer = blobClient.GetContainerReference("azuremailblobcontainer"); // Initialize table storage var tableClient = storageAccount.CreateCloudTableClient(); mailingListTable = tableClient.GetTableReference("mailinglist"); messageTable = tableClient.GetTableReference("message"); messagearchiveTable = tableClient.GetTableReference("messagearchive"); Trace.TraceInformation("WorkerB: Creating blob container, queue, tables, if they don't exist."); blobContainer.CreateIfNotExists(); sendEmailQueue.CreateIfNotExists(); subscribeQueue.CreateIfNotExists(); this.messageTable.CreateIfNotExists(); this.mailingListTable.CreateIfNotExists(); this.messagearchiveTable.CreateIfNotExists(); return base.OnStart(); }
public QueueMessageSource() { var account = MakeAccount(); queue = MakeQueue(account); queue.CreateIfNotExists(); }
public QueueHelper(string storageAccountConnectionString) : base(storageAccountConnectionString) { queueClient = base.StorageAccount.CreateCloudQueueClient(); subscribeQueue = queueClient.GetQueueReference(ConfigurationManager.AppSettings["QueueAzuremailsubscribequeue"]); subscribeQueue.CreateIfNotExists(); }
public QueueHelper(string connection, string queueName) { var account = CloudStorageAccount.Parse(connection); var retry = new LinearRetry(TimeSpan.FromSeconds(1), 3); queueClient = account.CreateCloudQueueClient(); queueClient.RetryPolicy = retry; queue = queueClient.GetQueueReference(queueName); queue.CreateIfNotExists(); }
public AzureQueue() { var cloudStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(Azure.StorageConnectionString)); var cloudQueueClient= cloudStorageAccount.CreateCloudQueueClient(); _cloudQueue= cloudQueueClient.GetQueueReference(CloudConfigurationManager.GetSetting(Azure.QueueName)); _cloudQueue.CreateIfNotExists(); }
static QueueStorageContext() { CloudStorageAccount account = CloudStorageAccount.Parse( System.Configuration.ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString); CloudQueueClient queueClient = account.CreateCloudQueueClient(); queue = queueClient.GetQueueReference(DetectionQueueName); queue.CreateIfNotExists(); }
public AzureStorageQueue(string name, string connectionString) { Name = name; ConnectionString = connectionString; var storageAccount = CloudStorageAccount.Parse(connectionString); var queueClient = storageAccount.CreateCloudQueueClient(); m_Queue = queueClient.GetQueueReference(name); m_Queue.CreateIfNotExists(); }
public PickCreator() { _storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); _queueClient = _storageAccount.CreateCloudQueueClient(); _queue = _queueClient.GetQueueReference("scaleoutsamplequeue"); _queue.CreateIfNotExists(); }
private static void CreateTestQueues() { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.ConnectionStrings["AzureWebJobsStorage"].ConnectionString); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); CloudQueue queue = queueClient.GetQueueReference("singleton-test"); queue.CreateIfNotExists(); _testQueue = queueClient.GetQueueReference("testqueue"); _testQueue.CreateIfNotExists(); }
/// <summary> /// Constructor /// </summary> /// <param name="queueName">The name of the queue to be managed</param> /// <param name="storageConnectionString">The connection string pointing to the storage account (this can be local or hosted in Windows Azure</param> public QueueStorageAsync(string queueName, string storageConnectionString) { Validate.QueueName(queueName, "queueName"); Validate.String(storageConnectionString, "storageConnectionString"); var cloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString); var cloudQueueClient = cloudStorageAccount.CreateCloudQueueClient(); cloudQueue = cloudQueueClient.GetQueueReference(queueName); cloudQueue.CreateIfNotExists(); }
public AzureQueueExt(string conectionString, string queueName, params Type[] types) { queueName = queueName.ToLower(); var storageAccount = CloudStorageAccount.Parse(conectionString); var queueClient = storageAccount.CreateCloudQueueClient(); _queue = queueClient.GetQueueReference(queueName); _queue.CreateIfNotExists(); RegisterTypes(types); }
private static void QueueInit() { const string queueName = "performance-demo"; CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["AzureStorageConnectionString"]); var cloudQueueClient = storageAccount.CreateCloudQueueClient(); _Queue = cloudQueueClient.GetQueueReference(queueName); _Queue.CreateIfNotExists(); _Watch = Stopwatch.StartNew(); }
public override bool OnStart() { _account = CloudStorageAccount.Parse( RoleEnvironment.GetConfigurationSettingValue("SiteMonitRConnectionString") ); _queueClient = _account.CreateCloudQueueClient(); _queue = _queueClient.GetQueueReference(new WebSiteQueueConfiguration().GetIncomingQueueName()); _queue.CreateIfNotExists(); return base.OnStart(); }
public override void Run() { storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); cmdQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlercmd"); urlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlerurl"); errorQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlererror"); lastTenUrlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("lasttenurlcrawled"); index = storageAccount.CreateCloudTableClient().GetTableReference("krawlerindex"); visisted = new HashSet<string>(); cmdQueue.CreateIfNotExists(); urlQueue.CreateIfNotExists(); errorQueue.CreateIfNotExists(); lastTenUrlQueue.CreateIfNotExists(); // This is a sample worker implementation. Replace with your logic. Trace.TraceInformation("WorkerRole entry point called", "Information"); Crawler crawler = new Crawler(); //cmdQueue.Clear(); //cmdQueue.AddMessage(new CloudQueueMessage("stop")); while (true) { Thread.Sleep(500); Trace.TraceInformation("Working", "Information"); CloudQueueMessage cmd = cmdQueue.PeekMessage(); if (cmd == null || cmd.AsString == "stop") { continue; } else if (cmd.AsString == "start") { CloudQueueMessage url = urlQueue.GetMessage(); if (!visisted.Contains(url.AsString)) { crawler.Crawl(url.AsString); visisted.Add(url.AsString); } urlQueue.DeleteMessage(url); } else if (cmd.AsString == "load" && urlQueue.PeekMessage() == null) { crawler.LoadQueue("http://www.cnn.com"); } else { continue; } } }
public void Init(Address address, bool transactional) { useTransactions = transactional; var queueName = AzureMessageQueueUtils.GetQueueName(address); azureQueue = Client.GetQueueReference(queueName); azureQueue.CreateIfNotExists(); if (PurgeOnStartup) { azureQueue.Clear(); } }
public WindowsAzureQueueSiteResultReceiver(IStorageQueueConfiguration configuration) { _configuration = configuration; var connectionString = _configuration.GetConnectionString(); var queueToReceiveFrom = _configuration.GetIncomingQueueName(); _client = CloudStorageAccount.Parse(connectionString).CreateCloudQueueClient(); if (!string.IsNullOrEmpty(queueToReceiveFrom)) { _incomingQueue = _client.GetQueueReference(queueToReceiveFrom); _incomingQueue.CreateIfNotExists(); } }
public LeaseExpirationTests() { RandomNameResolver nameResolver = new RandomNameResolver(); _config = new JobHostConfiguration() { NameResolver = nameResolver, TypeLocator = new FakeTypeLocator(typeof(LeaseExpirationTests)) }; CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_config.StorageConnectionString); _queueClient = storageAccount.CreateCloudQueueClient(); _queue = _queueClient.GetQueueReference(nameResolver.ResolveInString(TestQueueName)); _queue.CreateIfNotExists(); }
protected AzureStorageQueueClient(AzureStorageQueueEndpoint endpoint, IAzureStorageConfiguration storageConfiguration) { if (storageConfiguration == null) throw new ArgumentNullException("storageConfiguration"); endpoint.Validate(); storageConfiguration.Validate(); CloudStorageAccount = CloudStorageAccount.Parse(storageConfiguration.ConnectionString); CloudQueueClient = CloudStorageAccount.CreateCloudQueueClient(); CloudQueue = CloudQueueClient.GetQueueReference(endpoint.QueueName); CloudQueue.CreateIfNotExists(); }
private static void CreateTestQueues() { string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); CloudQueue queue = queueClient.GetQueueReference("singleton-test"); queue.CreateIfNotExists(); _testQueue = queueClient.GetQueueReference("testqueue"); _testQueue.CreateIfNotExists(); CloudQueue testQueue2 = queueClient.GetQueueReference("testqueue2"); testQueue2.CreateIfNotExists(); }
public JobQueue() { // var connectionString = RoleEnvironment.GetConfigurationSettingValue("PhotoAppStorage"); //var connectionString = ConfigurationManager.ConnectionStrings["PhotoAppStorage"].ConnectionString; //var account = CloudStorageAccount.Parse(connectionString); //var account = CloudStorageAccount.DevelopmentStorageAccount; var cred = new StorageCredentials("jholm", "/bVipQ2JxjWwYrZQfHmzhaBx1p1s8BoD/wX6VWOmg4/gpVo/aALrjsDUKqzXsFtc9utepPqe65NposrXt9YsyA=="); var account = new CloudStorageAccount(cred, true); var cloudQueueClient = account.CreateCloudQueueClient(); _cloudQueue = cloudQueueClient.GetQueueReference(_queueName); _cloudQueue.CreateIfNotExists(); }
/// <summary> /// Initializes a new instance of Producer using the given connectionSettings to connect to the Queue. /// </summary> /// <param name="connectionSettings"> /// The ConnectionSettings which define the Storage Account and Queue to use. /// </param> /// <param name="factory"> /// The JobFactory to use to serialize instances of IJob. /// </param> public Producer(ConnectionSettings connectionSettings, JobFactory factory) { if (null == connectionSettings) { throw new ArgumentNullException("connectionSettings"); } else if (null == factory) { throw new ArgumentNullException("factory"); } queue = connectionSettings.GetQueue(); queue.CreateIfNotExists(); this.connectionSettings = connectionSettings; this.factory = factory; }
public void Initialise(string queueName) { if (string.IsNullOrEmpty(queueName)) { throw new ArgumentException("QueueName cannot be null or empty", "queueName"); } string connStr = ConfigManager.ConfigManager.GetConfigurationSetting(ConfigManager.ConfigManager.DefaultQueueStorageConnString, string.Empty); _storageAccount = CloudStorageAccount.Parse(connStr); _queueStorage = _storageAccount.CreateCloudQueueClient(); _queue = _queueStorage.GetQueueReference(queueName); _queue.CreateIfNotExists(); }
public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 12; CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("AzureStorageConnectionString")); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); _client = queueClient.GetQueueReference("azqueue"); _client.CreateIfNotExists(); _database = ConfigurationManager.AppSettings["DBSelfLink"]; //_dichotomy = new Dichotomy(url, key); _loadBalance = new LoadBalance(); return base.OnStart(); }
/// <summary> /// Initializes a new instance of Consumer using the given connectionSettings to connect to the Queue. /// </summary> /// <param name="connectionSettings"> /// The ConnectionSettings which define the Storage Account and Queue to use. /// </param> /// <param name="factory"> /// The JobFactory to use to create instances of IJob. /// </param> /// <param name="settings"> /// The ConsumerSettings to use. If null, the default ConsumerSettings will be used. /// </param> public Consumer(ConnectionSettings connectionSettings, JobFactory factory, ConsumerSettings settings) { if (null == connectionSettings) { throw new ArgumentNullException("connectionSettings"); } else if (null == factory) { throw new ArgumentNullException("factory"); } queue = connectionSettings.GetQueue(); queue.CreateIfNotExists(); this.connectionSettings = connectionSettings; this.factory = factory; this.consumerSettings = settings ?? ConsumerSettings.CreateDefault(); }
public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 12; CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("Microsoft.Storage.ConnectionString")); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); _client = queueClient.GetQueueReference("azqueue"); _client.CreateIfNotExists(); _chatBackup = new ChatBackup(storageAccount); return base.OnStart(); }
public void ListQueuesSegmentedTest() { String prefix = "pagingqueuetest" + Guid.NewGuid(); ///Create 20 queues for (int i = 1; i <= 20; i++) { CloudQueue myqueue = DefaultQueueClient.GetQueueReference(prefix + i); myqueue.CreateIfNotExists(); } ///Segmented listing of queues. ///Return a page of 10 queues beginning with the specified prefix. ///Check with options and context as NULL QueueResultSegment resultSegment = DefaultQueueClient.ListQueuesSegmented(prefix, QueueListingDetails.None, 10, null, null, null); IEnumerable <CloudQueue> list = resultSegment.Results; int count = 0; foreach (CloudQueue item in list) { count++; item.Delete(); } Assert.AreEqual(10, count); Assert.IsNotNull(resultSegment.ContinuationToken); OperationContext context = new OperationContext(); QueueRequestOptions options = new QueueRequestOptions(); ///Check with options and context having some value QueueResultSegment resultSegment2 = DefaultQueueClient.ListQueuesSegmented(prefix, QueueListingDetails.None, 10, resultSegment.ContinuationToken, options, context); IEnumerable <CloudQueue> list2 = resultSegment2.Results; foreach (CloudQueue item in list2) { item.Delete(); } Assert.IsNull(resultSegment2.ContinuationToken); }
public void CloudQueueAddUpdateEncryptedEncodedMessage() { // Create the Key to be used for wrapping. SymmetricKey aesKey = new SymmetricKey("symencryptionkey"); CloudQueueClient client = GenerateCloudQueueClient(); string name = GenerateNewQueueName(); CloudQueue queue = client.GetQueueReference(name); try { queue.CreateIfNotExists(); byte[] messageBytes = new byte[100]; Random rand = new Random(); rand.NextBytes(messageBytes); string inputMessage = Convert.ToBase64String(messageBytes); CloudQueueMessage message = new CloudQueueMessage(inputMessage); queue.EncodeMessage = false; QueueEncryptionPolicy policy = new QueueEncryptionPolicy(aesKey, null); QueueRequestOptions options = new QueueRequestOptions() { EncryptionPolicy = policy }; // Add message queue.AddMessage(message, null, null, options, null); // Retrieve message CloudQueueMessage retrMessage = queue.GetMessage(null, options, null); Assert.AreEqual(inputMessage, retrMessage.AsString); } finally { queue.DeleteIfExists(); } }
public void CloudQueueAddEncrypted64KMessage() { // Create the Key to be used for wrapping. SymmetricKey aesKey = new SymmetricKey("symencryptionkey"); CloudQueueClient client = GenerateCloudQueueClient(); string name = GenerateNewQueueName(); CloudQueue queue = client.GetQueueReference(name); try { queue.CreateIfNotExists(); string inputMessage = new string('a', 64 * 1024); CloudQueueMessage message = new CloudQueueMessage(inputMessage); queue.EncodeMessage = false; QueueEncryptionPolicy policy = new QueueEncryptionPolicy(aesKey, null); QueueRequestOptions options = new QueueRequestOptions() { EncryptionPolicy = policy }; // Add message queue.AddMessage(message, null, null, null, null); // Add encrypted Message TestHelper.ExpectedException <ArgumentException>( () => queue.AddMessage(message, null, null, options, null), "Adding an encrypted message that exceeds message limits should throw"); } finally { queue.DeleteIfExists(); } }
public void CloudQueueMessageEncryptionWithStrictMode() { // Create the Key to be used for wrapping. SymmetricKey aesKey = new SymmetricKey("symencryptionkey"); // Create the resolver to be used for unwrapping. DictionaryKeyResolver resolver = new DictionaryKeyResolver(); resolver.Add(aesKey); CloudQueueClient client = GenerateCloudQueueClient(); string name = GenerateNewQueueName(); CloudQueue queue = client.GetQueueReference(name); try { queue.CreateIfNotExists(); string messageStr = Guid.NewGuid().ToString(); CloudQueueMessage message = new CloudQueueMessage(messageStr); QueueEncryptionPolicy policy = new QueueEncryptionPolicy(aesKey, null); // Add message with policy. QueueRequestOptions createOptions = new QueueRequestOptions() { EncryptionPolicy = policy }; createOptions.RequireEncryption = true; queue.AddMessage(message, null, null, createOptions, null); // Set policy to null and add message while RequireEncryption flag is still set to true. This should throw. createOptions.EncryptionPolicy = null; TestHelper.ExpectedException <InvalidOperationException>( () => queue.AddMessage(message, null, null, createOptions, null), "Not specifying a policy when RequireEnryption is set to true should throw."); // Retrieve message QueueEncryptionPolicy retrPolicy = new QueueEncryptionPolicy(null, resolver); QueueRequestOptions retrieveOptions = new QueueRequestOptions() { EncryptionPolicy = retrPolicy }; retrieveOptions.RequireEncryption = true; CloudQueueMessage retrMessage = queue.GetMessage(null, retrieveOptions, null); // Update message with plain text. string updatedMessage = Guid.NewGuid().ToString("N"); retrMessage.SetMessageContent(updatedMessage); queue.UpdateMessage(retrMessage, TimeSpan.FromSeconds(0), MessageUpdateFields.Content | MessageUpdateFields.Visibility); // Retrieve updated message with RequireEncryption flag but no metadata on the service. This should throw. TestHelper.ExpectedException <StorageException>( () => queue.GetMessage(null, retrieveOptions, null), "Retrieving with RequireEncryption set to true and no metadata on the service should fail."); // Set RequireEncryption to false and retrieve. retrieveOptions.RequireEncryption = false; queue.GetMessage(null, retrieveOptions, null); } finally { queue.DeleteIfExists(); } }