public sealed override void DoStuff()
{
using (MongoQueue <TaskMessage> queue = new MongoQueue <TaskMessage>(ConfigurationManager.AppSettings["mongodbqueue"], 1000000))
{
if (Strategy == MiningStrategy.UntilLastPageNum)
{
for (int i = 1; i <= LastPageNum; i += Step)
{
string url = string.Format(ListPageUrl, i);
var msg = CreatePageTask(this.Name, url);
queue.Send(msg);
}
}
else
{
int i = 1;
string url = string.Format(ListPageUrl, i);
HttpRequestContentExtractor ce = new HttpRequestContentExtractor(HttpMethod.GET, null);
while (ce.Goto(url))
{
var msg = CreatePageTask(this.Name, url);
queue.Send(msg);
i += Step;
url = string.Format(ListPageUrl, i);
}
}
}
}
public static void Main()
{
Console.WriteLine("QueryWorkerApp starting...");
CloudFoundryMongoBinder binder = new CloudFoundryMongoBinder();
MongoQueue <string> collectorQueue = new MongoQueue <string>(binder.Url, binder.DatabaseName, "collector_queue", 32000);
MongoQueue <string> workerQueue = new MongoQueue <string>(binder.Url, binder.DatabaseName, "worker_queue", 32000);
try
{
while (true)
{
Console.WriteLine("Waiting...");
string result = workerQueue.Receive();
Console.WriteLine("Received " + result);
collectorQueue.Send(binder.AppPort + ":" + result.ToUpper());
Console.WriteLine("Back to top of loop");
}
}
catch (Exception e)
{
Console.WriteLine("Query worker exiting after exception: " + e);
}
}
public override void DoStuff()
{
using (MongoQueue <TaskMessage> queue = new MongoQueue <TaskMessage>(ConfigurationManager.AppSettings["mongodbqueue"], 1000000))
{
for (int i = StartPageNo; i <= EndPageNo; i++)
{
var msg = CreatePageTask(this.Name, string.Format(UrlTemplate, i));
queue.Send(msg);
}
}
}
static void Main(string[] args)
{
var options = new Options();
if (!CommandLine.Parser.Default.ParseArguments(args, options))
{
return;
}
using (MongoQueue <TaskMessage> queue = new MongoQueue <TaskMessage>(ConfigurationManager.AppSettings["mongodbqueue"], 100000))
{
queue.Send(CreateMsg(options));
}
}