예제 #1
0
        private void PushToRabbit(PageCrawlResult result)
        {
            var exchangeName = "crawled_notifications";
            var rabbitHost   = "rabbit";

            var factory = new ConnectionFactory()
            {
                HostName = rabbitHost
            };

            using (var connection = factory.CreateConnection())
                using (var channel = connection.CreateModel())
                {
                    channel.ExchangeDeclare(exchange: exchangeName, type: "fanout");

                    var message = new PageCrawledNotification()
                    {
                        Html = result.Html,
                        Uri  = result.Url
                    };
                    var messageSerialized = JsonConvert.SerializeObject(message);
                    var body = Encoding.UTF8.GetBytes(messageSerialized);
                    channel.BasicPublish(exchange: exchangeName,
                                         routingKey: "",
                                         basicProperties: null,
                                         body: body);
                    _logger.LogInformation($"Published notification of page crawl of uri [{result.Url}] to exchange [{exchangeName}] on host [{rabbitHost}]");
                }
        }
예제 #2
0
        public void Save(PageCrawlResult page)
        {
            _logger.LogInformation($"Saving crawl result of page [{page.Url}] to the database.");
            var db         = _redisProvider.GetDatabase();
            var serialized = JsonConvert.SerializeObject(page);

            db.StringSet(page.Url, serialized);
        }
예제 #3
0
 public void Notify(PageCrawlResult result)
 {
     while (true)
     {
         try
         {
             PushToRabbit(result);
             break;
         }
         catch (Exception)
         {
             _logger.LogInformation("Failed to push to rabbit. Retrying.");
             Thread.Sleep(1000);
         }
     }
 }