public override void Run() { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); datatable = tableClient.GetTableReference("datatable"); datatable.CreateIfNotExists(); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); urlQueue = queueClient.GetQueueReference("urlqueue"); urlQueue.CreateIfNotExists(); adminQueue = queueClient.GetQueueReference("adminqueue"); adminQueue.CreateIfNotExists(); ramCounter = new System.Diagnostics.PerformanceCounter("Memory", "Available MBytes"); cpuCounter = new System.Diagnostics.PerformanceCounter(); cpuCounter.CategoryName = "Processor"; cpuCounter.CounterName = "% Processor Time"; cpuCounter.InstanceName = "_Total"; crawler = new Crawler(); while (true) { Thread.Sleep(50); //If there is something in the admin queue if (adminQueue.PeekMessage() != null) { CloudQueueMessage message = adminQueue.GetMessage(); crawler.handleAdminMessage(message); adminQueue.DeleteMessage(message); } if (urlQueue.PeekMessage() != null) { crawler.crawlingPhase(urlQueue.GetMessage()); } } }
private static void QueueBase64EncodingTestDownloadMessageAndVerify(CloudQueue queue, CloudQueue queueRefWithoutBase64Encoding, byte[] originalData) { // Assumption: 2 of the same messages have been added // If the message was uploaded with Base64Encoding, this function will also retrieve the message without Base64 encoding. CloudQueueMessage readBack = queue.GetMessage(); if (!CompareByteArray(originalData, readBack.AsBytes)) { string orignalData = PrintByteArray(originalData, "OriginalData"); string returnedData = PrintByteArray(readBack.AsBytes, "ReturnedData"); Assert.Fail("Data read back from server doesn't match the original data. \r\n{0}\r\n{1}", orignalData, returnedData); } queue.DeleteMessage(readBack); if (queue.EncodeMessage) { CloudQueueMessage readBackWithoutBase64Encoding = queueRefWithoutBase64Encoding.GetMessage(); byte[] returnedDataWithoutBase64Encoding = Convert.FromBase64String(readBackWithoutBase64Encoding.AsString); if (!CompareByteArray(originalData, returnedDataWithoutBase64Encoding)) { string orignalData = PrintByteArray(originalData, "OriginalData"); string returnedData = PrintByteArray(returnedDataWithoutBase64Encoding, "ReturnedDataWithoutBase64Encoding"); Assert.Fail("Data read back from server doesn't match the original data. \r\n{0}\r\n{1}", orignalData, returnedData); } queueRefWithoutBase64Encoding.DeleteMessage(readBackWithoutBase64Encoding); } else { readBack = queue.GetMessage(); queue.DeleteMessage(readBack); } }
private static void QueueBase64EncodingTestDownloadMessageAndVerify(CloudQueue queue, CloudQueue queueRefWithoutBase64Encoding, string originalMessage) { // Assumption: 2 of the same messages have been added // If the message was uploaded with Base64Encoding, this function will also retrieve the message without Base64 encoding. CloudQueueMessage readBack = queue.GetMessage(); Assert.AreEqual<string>(originalMessage, readBack.AsString); queue.DeleteMessage(readBack); if (queue.EncodeMessage) { CloudQueueMessage readBackWithoutBase64Encoding = queueRefWithoutBase64Encoding.GetMessage(); string decodedMessage = Encoding.UTF8.GetString(Convert.FromBase64String(readBackWithoutBase64Encoding.AsString)); Assert.AreEqual<string>(originalMessage, decodedMessage); queueRefWithoutBase64Encoding.DeleteMessage(readBackWithoutBase64Encoding); } else { readBack = queue.GetMessage(); queue.DeleteMessage(readBack); } }
private void DeleteSourceMessages(CloudQueue sourceQueue, IList<CloudQueueMessage> inboundMessages, Stopwatch stopWatch) { inboundMessages.ForEach(inMsg => sourceQueue.DeleteMessage(inMsg)); this.LogInformation("Done deleting {0} messages from {1} queue. Elapsed {2}.", inboundMessages.Count, sourceQueue.Name, stopWatch.Elapsed); }
internal void Acknowledge(CloudQueue queue) { queue.DeleteMessage(m_AzureMessage); }
public override void Run() { storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); cmdQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlercmd"); urlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlerurl"); errorQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlererror"); lastTenUrlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("lasttenurlcrawled"); index = storageAccount.CreateCloudTableClient().GetTableReference("krawlerindex"); visisted = new HashSet<string>(); cmdQueue.CreateIfNotExists(); urlQueue.CreateIfNotExists(); errorQueue.CreateIfNotExists(); lastTenUrlQueue.CreateIfNotExists(); // This is a sample worker implementation. Replace with your logic. Trace.TraceInformation("WorkerRole entry point called", "Information"); Crawler crawler = new Crawler(); //cmdQueue.Clear(); //cmdQueue.AddMessage(new CloudQueueMessage("stop")); while (true) { Thread.Sleep(500); Trace.TraceInformation("Working", "Information"); CloudQueueMessage cmd = cmdQueue.PeekMessage(); if (cmd == null || cmd.AsString == "stop") { continue; } else if (cmd.AsString == "start") { CloudQueueMessage url = urlQueue.GetMessage(); if (!visisted.Contains(url.AsString)) { crawler.Crawl(url.AsString); visisted.Add(url.AsString); } urlQueue.DeleteMessage(url); } else if (cmd.AsString == "load" && urlQueue.PeekMessage() == null) { crawler.LoadQueue("http://www.cnn.com"); } else { continue; } } }