private void PushToRabbit(PageCrawlResult result)
{
var exchangeName = "crawled_notifications";
var rabbitHost = "rabbit";
var factory = new ConnectionFactory()
{
HostName = rabbitHost
};
using (var connection = factory.CreateConnection())
using (var channel = connection.CreateModel())
{
channel.ExchangeDeclare(exchange: exchangeName, type: "fanout");
var message = new PageCrawledNotification()
{
Html = result.Html,
Uri = result.Url
};
var messageSerialized = JsonConvert.SerializeObject(message);
var body = Encoding.UTF8.GetBytes(messageSerialized);
channel.BasicPublish(exchange: exchangeName,
routingKey: "",
basicProperties: null,
body: body);
_logger.LogInformation($"Published notification of page crawl of uri [{result.Url}] to exchange [{exchangeName}] on host [{rabbitHost}]");
}
}
public void Save(PageCrawlResult page)
{
_logger.LogInformation($"Saving crawl result of page [{page.Url}] to the database.");
var db = _redisProvider.GetDatabase();
var serialized = JsonConvert.SerializeObject(page);
db.StringSet(page.Url, serialized);
}
public void Notify(PageCrawlResult result)
{
while (true)
{
try
{
PushToRabbit(result);
break;
}
catch (Exception)
{
_logger.LogInformation("Failed to push to rabbit. Retrying.");
Thread.Sleep(1000);
}
}
}
