public void FrameworkMessage(ISchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, byte[] data)
{
var message = JsonHelper.Deserialize<Message> (data);
switch (message.Type) {
case "CrawlResult":
var crawlResult = JsonHelper.Deserialize<CrawlResultMessage> (message.Body);
Console.WriteLine ($"Framework message <CrawlResult>: got {crawlResult.Links.Length} links from url '{crawlResult.Url}'.");
foreach (var link in crawlResult.Links) {
if (_crawled.Contains (link))
continue;
_crawlQueue.Enqueue (link);
_renderQueue.Enqueue (link);
}
// update edges: url -> links
var edges = _crawlResults.GetOrAdd (crawlResult.Url, x => new List<string> ());
edges.AddRange (crawlResult.Links);
// empty edge list for links
foreach (var link in crawlResult.Links)
_crawlResults.GetOrAdd (link, x => new List<string> ());
break;
case "RenderResult":
var renderResult = JsonHelper.Deserialize<RenderResultMessage> (message.Body);
Console.WriteLine ($"Framework message <RenderResult>: saved '{renderResult.FileName}' for url '{renderResult.Url}'.");
_renderResults [renderResult.Url] = renderResult.FileName;
break;
default:
Console.WriteLine ($"Unrecognized message type: '{message.Type}'");
break;
}
}
public void Error(ISchedulerDriver driver, string message)
{
Console.WriteLine($"Error: '{message}'.");
}
public void ExecutorLost(ISchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, int status)
{
}
public void StatusUpdate(ISchedulerDriver driver, TaskStatus status)
{
if (status.state.IsTerminal())
{
Console.WriteLine($"Status update: task '{status.task_id.value}' has terminated with state '{status.state}'.");
var finishedTasksCount = Interlocked.Increment(ref _finishedTasksCount);
if (finishedTasksCount == MaxTasksToRun)
{
Console.WriteLine("Reached the max number of tasks to run. Stopping...");
var dotWritePath = Path.Combine(_outputDir, "result.dot");
DotHelper.Write(dotWritePath, _crawlResults, _renderResults);
driver.Stop();
}
}
else
{
Console.WriteLine($"Status update: task '{status.task_id.value}' is in state '{status.state}'.");
}
}
public void Disconnected(ISchedulerDriver driver)
{
}
public void SlaveLost(ISchedulerDriver driver, SlaveID slaveId)
{
}
public void ResourceOffers(ISchedulerDriver driver, IEnumerable<Offer> offers)
{
foreach (var offer in offers)
{
var tasks = new List<TaskInfo>();
var resourcesCounter = new ResourcesCounter(offer);
bool done;
do
{
done = true;
string renderUrl;
if (resourcesCounter.HasRenderTaskResources() && _renderQueue.TryDequeue(out renderUrl))
{
tasks.Add(GetRenderTaskInfo(offer, ++_launchedTasks, renderUrl));
resourcesCounter.SubstractRenderResources();
done = false;
}
string crawlUrl;
if (resourcesCounter.HasCrawlTaskResources() && _crawlQueue.TryDequeue(out crawlUrl))
{
tasks.Add(GetCrawlTaskInfo(offer, ++_launchedTasks, crawlUrl));
resourcesCounter.SubstractCrawlResources();
_crawled.Add(crawlUrl);
done = false;
}
} while (!done);
if (tasks.Any ()) {
driver.LaunchTasks (new[] { offer.id }, tasks);
}
else
driver.DeclineOffer(offer.id);
}
}
public void Reregistered(ISchedulerDriver driver, MasterInfo masterInfo)
{
}
public void Registered(ISchedulerDriver driver, FrameworkID frameworkId, MasterInfo masterInfo)
{
Console.WriteLine($"Registered with Mesos master. FrameworkId='{frameworkId.value}'.");
}
public void OfferRescinded(ISchedulerDriver driver, OfferID offerId)
{
}
