public override void Run()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
datatable = tableClient.GetTableReference("datatable");
datatable.CreateIfNotExists();
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
urlQueue = queueClient.GetQueueReference("urlqueue");
urlQueue.CreateIfNotExists();
adminQueue = queueClient.GetQueueReference("adminqueue");
adminQueue.CreateIfNotExists();
ramCounter = new System.Diagnostics.PerformanceCounter("Memory", "Available MBytes");
cpuCounter = new System.Diagnostics.PerformanceCounter();
cpuCounter.CategoryName = "Processor";
cpuCounter.CounterName = "% Processor Time";
cpuCounter.InstanceName = "_Total";
crawler = new Crawler();
while (true)
{
Thread.Sleep(50);
//If there is something in the admin queue
if (adminQueue.PeekMessage() != null)
{
CloudQueueMessage message = adminQueue.GetMessage();
crawler.handleAdminMessage(message);
adminQueue.DeleteMessage(message);
}
if (urlQueue.PeekMessage() != null)
{
crawler.crawlingPhase(urlQueue.GetMessage());
}
}
}
private static void QueueBase64EncodingTestDownloadMessageAndVerify(CloudQueue queue, CloudQueue queueRefWithoutBase64Encoding, byte[] originalData)
{
// Assumption: 2 of the same messages have been added
// If the message was uploaded with Base64Encoding, this function will also retrieve the message without Base64 encoding.
CloudQueueMessage readBack = queue.GetMessage();
if (!CompareByteArray(originalData, readBack.AsBytes))
{
string orignalData = PrintByteArray(originalData, "OriginalData");
string returnedData = PrintByteArray(readBack.AsBytes, "ReturnedData");
Assert.Fail("Data read back from server doesn't match the original data. \r\n{0}\r\n{1}", orignalData, returnedData);
}
queue.DeleteMessage(readBack);
if (queue.EncodeMessage)
{
CloudQueueMessage readBackWithoutBase64Encoding = queueRefWithoutBase64Encoding.GetMessage();
byte[] returnedDataWithoutBase64Encoding = Convert.FromBase64String(readBackWithoutBase64Encoding.AsString);
if (!CompareByteArray(originalData, returnedDataWithoutBase64Encoding))
{
string orignalData = PrintByteArray(originalData, "OriginalData");
string returnedData = PrintByteArray(returnedDataWithoutBase64Encoding, "ReturnedDataWithoutBase64Encoding");
Assert.Fail("Data read back from server doesn't match the original data. \r\n{0}\r\n{1}", orignalData, returnedData);
}
queueRefWithoutBase64Encoding.DeleteMessage(readBackWithoutBase64Encoding);
}
else
{
readBack = queue.GetMessage();
queue.DeleteMessage(readBack);
}
}
private static void QueueBase64EncodingTestDownloadMessageAndVerify(CloudQueue queue, CloudQueue queueRefWithoutBase64Encoding, string originalMessage)
{
// Assumption: 2 of the same messages have been added
// If the message was uploaded with Base64Encoding, this function will also retrieve the message without Base64 encoding.
CloudQueueMessage readBack = queue.GetMessage();
Assert.AreEqual<string>(originalMessage, readBack.AsString);
queue.DeleteMessage(readBack);
if (queue.EncodeMessage)
{
CloudQueueMessage readBackWithoutBase64Encoding = queueRefWithoutBase64Encoding.GetMessage();
string decodedMessage = Encoding.UTF8.GetString(Convert.FromBase64String(readBackWithoutBase64Encoding.AsString));
Assert.AreEqual<string>(originalMessage, decodedMessage);
queueRefWithoutBase64Encoding.DeleteMessage(readBackWithoutBase64Encoding);
}
else
{
readBack = queue.GetMessage();
queue.DeleteMessage(readBack);
}
}
private void DeleteSourceMessages(CloudQueue sourceQueue, IList<CloudQueueMessage> inboundMessages, Stopwatch stopWatch)
{
inboundMessages.ForEach(inMsg => sourceQueue.DeleteMessage(inMsg));
this.LogInformation("Done deleting {0} messages from {1} queue. Elapsed {2}.", inboundMessages.Count, sourceQueue.Name, stopWatch.Elapsed);
}
internal void Acknowledge(CloudQueue queue)
{
queue.DeleteMessage(m_AzureMessage);
}
public override void Run()
{
storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
cmdQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlercmd");
urlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlerurl");
errorQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("krawlererror");
lastTenUrlQueue = storageAccount.CreateCloudQueueClient().GetQueueReference("lasttenurlcrawled");
index = storageAccount.CreateCloudTableClient().GetTableReference("krawlerindex");
visisted = new HashSet<string>();
cmdQueue.CreateIfNotExists();
urlQueue.CreateIfNotExists();
errorQueue.CreateIfNotExists();
lastTenUrlQueue.CreateIfNotExists();
// This is a sample worker implementation. Replace with your logic.
Trace.TraceInformation("WorkerRole entry point called", "Information");
Crawler crawler = new Crawler();
//cmdQueue.Clear();
//cmdQueue.AddMessage(new CloudQueueMessage("stop"));
while (true)
{
Thread.Sleep(500);
Trace.TraceInformation("Working", "Information");
CloudQueueMessage cmd = cmdQueue.PeekMessage();
if (cmd == null || cmd.AsString == "stop")
{
continue;
}
else if (cmd.AsString == "start")
{
CloudQueueMessage url = urlQueue.GetMessage();
if (!visisted.Contains(url.AsString))
{
crawler.Crawl(url.AsString);
visisted.Add(url.AsString);
}
urlQueue.DeleteMessage(url);
}
else if (cmd.AsString == "load" && urlQueue.PeekMessage() == null)
{
crawler.LoadQueue("http://www.cnn.com");
}
else
{
continue;
}
}
}