static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); _logger = new LogWrapper(); // Loading Configuration _logger.LogMessage("Reading Configuration"); LoadConfiguration(); // AWS Queue Handler _logger.LogMessage("Initializing Queues"); AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret); // Step 1 - Trying to obtain the root page html (source of all the apps) var rootPageResponse = httpClient.GetRootPage(); // Sanity Check if (String.IsNullOrWhiteSpace(rootPageResponse)) { _logger.LogMessage("Error obtaining Root Page HTMl - Aborting", "Timeout Error"); return; } // Step 2 - Extracting Category Urls from the Root Page and queueing their Urls foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse)) { // Logging Feedback _logger.LogMessage("Queueing Category : " + categoryUrl); // Queueing Category Urls sqsWrapper.EnqueueMessage(categoryUrl); } _logger.LogMessage("End of Bootstrapping phase"); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); // Setting Up Log LogSetup.InitializeLog("Apple_Store_Crawler.log", "info"); _logger = LogManager.GetCurrentClassLogger(); // Starting Flow _logger.Info("Worker Started"); // Loading Configuration _logger.Info("Reading Configuration"); LoadConfiguration(); // Control Variable (Bool - Should the process use proxies? ) bool shouldUseProxies = false; // Checking for the need to use proxies if (args != null && args.Length == 1) { // Setting flag to true shouldUseProxies = true; // Loading proxies from .txt received as argument String fPath = args[0]; // Sanity Check if (!File.Exists(fPath)) { _logger.Fatal("Couldnt find proxies on path : " + fPath); System.Environment.Exit(-100); } // Reading Proxies from File string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8")); try { // Actual Load of Proxies ProxiesLoader.Load(fLines.ToList()); } catch (Exception ex) { _logger.Fatal(ex); System.Environment.Exit(-101); } } // AWS Queue Handler _logger.Info("Initializing Queues"); AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret); // Step 1 - Trying to obtain the root page html (source of all the apps) var rootPageResponse = httpClient.GetRootPage(shouldUseProxies); // Sanity Check if (String.IsNullOrWhiteSpace(rootPageResponse)) { _logger.Info("Error obtaining Root Page HTMl - Aborting", "Timeout Error"); return; } // Step 2 - Extracting Category Urls from the Root Page and queueing their Urls foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse)) { // Logging Feedback _logger.Info("Queueing Category : " + categoryUrl); // Queueing Category Urls sqsWrapper.EnqueueMessage(categoryUrl); } _logger.Info("End of Bootstrapping phase"); }