private static bool InitializeMongo()
{
MongoUtilsTopicObj = new MongoDBUtils <Topic>(Config.MongoUser, Config.MongoPassword, Config.MongoAddress, Config.MongoDatabase);
MongoUtilsCommentObj = new MongoDBUtils <Comment>(Config.MongoUser, Config.MongoPassword, Config.MongoAddress, Config.MongoDatabase);
MongoUtilsUserObj = new MongoDBUtils <User>(Config.MongoUser, Config.MongoPassword, Config.MongoAddress, Config.MongoDatabase);
// Sanit Check
if (!MongoUtilsTopicObj.IsValidMongoData(Config))
{
return(false);
}
// Invalid Collection?
if (!MongoUtilsTopicObj.CollectionExistsAsync(Config.MongoCollection).Result ||
!MongoUtilsCommentObj.CollectionExistsAsync(CommentsCollection).Result ||
!MongoUtilsUserObj.CollectionExistsAsync(UsersCollection).Result)
{
return(false);
}
// Open the Connection
MongoUtilsTopicObj.GetCollection(Config.MongoCollection);
MongoUtilsCommentObj.GetCollection(CommentsCollection);
MongoUtilsUserObj.GetCollection(UsersCollection);
return(true);
}
private static void Execute(Logger logger)
{
logger.Info("Start");
MSMQUtils MSMQ = new MSMQUtils();
// Initialization WebRequests
WebRequests client = new WebRequests();
SharedLibrary.Utils.WebRequestsUtils.InitializeWebRequest(Config.InitialUrl, out client, Config);
// Get Request
string htmlResponse = SharedLibrary.Utils.WebRequestsUtils.Get(ref client, logger, Config.InitialUrl);
// Checking if html response is valid
if (String.IsNullOrWhiteSpace(htmlResponse))
{
logger.Fatal("HtmlResponse is null or empty");
return;
}
client.Dispose();
// Loading htmlResponse into HtmlDocument
HtmlDocument map = new HtmlDocument();
map.LoadHtml(htmlResponse);
//Scrape Urls
logger.Trace("Scraping section urls...");
List <Section> sections = ScrapeSections(map);
// Check if any section was processed
if (sections == null || sections.Count == 0)
{
logger.Fatal("Couldn't build any section object. Aborting program");
goto Exit;
}
// Let's save the section's object
if (!String.IsNullOrWhiteSpace(Config.TargetQueue) && MongoUtilsObj.IsValidMongoData(Config))
{
// Send messages to Queue
logger.Trace("Sending message to configuration queue...");
SendMessage(Config.TargetQueue, sections);
logger.Trace("Sending message to MongoCollection...");
SendMessage(sections);
}
else
{
logger.Fatal("Error to save section's object. You need to check if there is something wrong with the information in the mongo/queue fields in the input file.");
}
Exit:
logger.Info("End");
}
private static bool InitializeMongo()
{
MongoUtilsObj = new MongoDBUtils <Section>(Config.MongoUser, Config.MongoPassword, Config.MongoAddress, Config.MongoDatabase);
// Sanit Check
if (!MongoUtilsObj.IsValidMongoData(Config))
{
return(false);
}
// Is There no collection?
if (!MongoUtilsObj.CollectionExistsAsync(Config.MongoCollection).Result)
{
MongoUtilsObj.CreateCollection(Config.MongoCollection);
}
return(true);
}