private static void SendMessage(string queuename, List <Section> sections)
{
MSMQUtils MSMQ = new MSMQUtils();
// Trying to open the queue
MessageQueue queue = MSMQ.OpenOrCreatePrivateQueue(queuename, typeof(SectionsParser).Namespace);
// Sanit check
if (queue == null)
{
logger.Fatal("Error to open a private queue. The field \"queue\" is null.");
return;
}
// Iterate over all sections
foreach (Section section in sections)
{
logger.Trace("Sending " + section.Title + " to " + queuename + " queue");
string serializedSection = Utils.Compress(JsonConvert.SerializeObject(section));
queue.Send(serializedSection);
}
queue.Dispose();
}
private static void SaveConfigurationFields()
{
MSMQUtils MSMQ = new MSMQUtils();
// Trying to open the queue
MessageQueue queue = MSMQ.OpenOrCreatePrivateQueue(Config.WebRequestConfigQueue, typeof(SectionsParser).Namespace);
// Sanit check
if (queue == null)
{
logger.Fatal("Error to open a private WebConfigQueue. The field \"queue\" is null.");
return;
}
// Is there content in the queue? Let's remove it
MSMQ.DeleteContentPrivateQueue(Config.WebRequestConfigQueue);
// Serialize the objectc to take up less space
string serializedConfig = Utils.Compress(JsonConvert.SerializeObject(Config));
Message message = new Message();
message.TimeToBeReceived = new TimeSpan(3 * 24, 0, 0);
message.Body = serializedConfig;
queue.Send(message);
}
private static Topic ReadQueue(string queuename)
{
MSMQUtils MSMQ = new MSMQUtils();
Topic topic = null;
try
{
// Trying to read the queue
object response = MSMQ.ReadPrivateQueue(queuename, timeoutInMinutes: 1);
string json = (string)response;
topic = JsonConvert.DeserializeObject <Topic>(Utils.Decompress(json));
}
catch (MessageQueueException mqex)
{
logger.Fatal(mqex, mqex.Message);
topic = null;
}
catch (Exception ex)
{
logger.Fatal("Error in ReadQueue Method. Message.: {0} ", ex.Message);
topic = null;
}
return(topic);
}
private static void SendMessage(string queuename, List <Comment> comments)
{
MSMQUtils MSMQ = new MSMQUtils();
// Trying to open the queue
MessageQueue queue = MSMQ.OpenOrCreatePrivateQueue(queuename, typeof(CommentsParser).Namespace);
// Sanit check
if (queue == null)
{
logger.Fatal("Error to open a private queue. The field \"queue\" is null.");
return;
}
// Iterate over all comments
foreach (Comment comment in comments)
{
var compactComment = new
{
Author = comment.Author
};
string serializedCompactComment = Utils.Compress(JsonConvert.SerializeObject(compactComment));
queue.Send(serializedCompactComment);
}
queue.Dispose();
}
private static void SendMessage(string queuename, List <Topic> topics)
{
MSMQUtils MSMQ = new MSMQUtils();
// Trying to open the queue
MessageQueue queue = MSMQ.OpenOrCreatePrivateQueue(queuename, typeof(TopicsParser).Namespace);
// Sanit check
if (queue == null)
{
logger.Fatal("Error to open a private queue. The field \"queue\" is null.");
return;
}
// Iterate over all topics
foreach (Topic topic in topics)
{
var compactTopic = new { Title = topic.Title,
Url = topic.Url };
string serializedCompactTopic = Utils.Compress(JsonConvert.SerializeObject(compactTopic));
queue.Send(serializedCompactTopic);
}
queue.Dispose();
}
private static void Execute(Logger logger)
{
logger.Info("Start");
MSMQUtils MSMQ = new MSMQUtils();
// Initialization WebRequests
WebRequests client = new WebRequests();
SharedLibrary.Utils.WebRequestsUtils.InitializeWebRequest(Config.InitialUrl, out client, Config);
// Get Request
string htmlResponse = SharedLibrary.Utils.WebRequestsUtils.Get(ref client, logger, Config.InitialUrl);
// Checking if html response is valid
if (String.IsNullOrWhiteSpace(htmlResponse))
{
logger.Fatal("HtmlResponse is null or empty");
return;
}
client.Dispose();
// Loading htmlResponse into HtmlDocument
HtmlDocument map = new HtmlDocument();
map.LoadHtml(htmlResponse);
//Scrape Urls
logger.Trace("Scraping section urls...");
List <Section> sections = ScrapeSections(map);
// Check if any section was processed
if (sections == null || sections.Count == 0)
{
logger.Fatal("Couldn't build any section object. Aborting program");
goto Exit;
}
// Let's save the section's object
if (!String.IsNullOrWhiteSpace(Config.TargetQueue) && MongoUtilsObj.IsValidMongoData(Config))
{
// Send messages to Queue
logger.Trace("Sending message to configuration queue...");
SendMessage(Config.TargetQueue, sections);
logger.Trace("Sending message to MongoCollection...");
SendMessage(sections);
}
else
{
logger.Fatal("Error to save section's object. You need to check if there is something wrong with the information in the mongo/queue fields in the input file.");
}
Exit:
logger.Info("End");
}
public static void Main(string[] args)
{
// Loading New Logger
logger = LogManager.GetCurrentClassLogger();
//Initialization AppConfig
InitializeAppConfig();
try
{
// Get Content from Configuration Queue
logger.Debug("Getting Content from Configuration Queue...");
Config = MSMQUtils.GetContentConfigurationQueue(ConfigurationQueueName);
}
catch (MessageQueueException mqex)
{
logger.Fatal(mqex, mqex.Message);
goto Exit;
}
// Sanit Check
if (Config == null)
{
logger.Fatal("Error to process the Configuration Queue.");
goto Exit;
}
// Initialization Mongo
logger.Debug("Initializing MongoDB...");
if (!InitializeMongo())
{
logger.Fatal("Error parsing Mongo variables! Aborting...");
goto Exit;
}
try
{
// Main Method
Execute(logger);
}
catch (Exception ex)
{
logger.Fatal("General Exception. \"Execute\" Method. Message.: {0}", ex.Message);
goto Exit;
}
Exit:
logger.Info("Press any key...");
Console.ReadKey();
}
