Пример #1
0
        public AppleStoreAppModel ParseAppPage(string htmlResponse)
        {
            // Creating HTML Map based on the html response
            HtmlDocument map = new HtmlDocument();

            map.LoadHtml(htmlResponse);

            // Instantiating Empty Parsed App
            AppleStoreAppModel parsedApp = new AppleStoreAppModel();

            // Reaching nodes of interest
            parsedApp.name          = GetNodeValue(map, Consts.XPATH_TITLE).Trim();
            parsedApp.developerName = GetAppDeveloperName(map);
            parsedApp.developerUrl  = GetDeveloperUrl(map);
            parsedApp.price         = GetAppPrice(map);
            parsedApp.isFree        = parsedApp.price == 0.0 ? true : false;
            parsedApp.category      = GetAppCategory(map);
            parsedApp.updateDate    = GetAppUpdateDate(map);
            //parsedApp.description     = GetAppDescription (map); //TODO: Figure out how to get description on a separated request
            parsedApp.version           = GetAppVersion(map);
            parsedApp.size              = GetAppSize(map);
            parsedApp.thumbnailUrl      = GetThumbnailUrl(map);
            parsedApp.languages         = GetLanguages(map);
            parsedApp.compatibility     = GetCompatibility(map);
            parsedApp.minimumAge        = GetMinimumAge(map);
            parsedApp.ageRatingReasons  = GetRatingReasons(map);
            parsedApp.rating            = GetRatings(map);
            parsedApp.topInAppPurchases = GetInAppPurchases(map);


            return(parsedApp);
        }
Пример #2
0
        static void Main(string[] args)
        {
            // Creating Needed Instances
            RequestsHandler httpClient = new RequestsHandler();
            AppStoreParser  parser     = new AppStoreParser();

            // Loading Configuration
            LogSetup.InitializeLog("Apple_Store_Urls_Worker.log", "info");
            _logger = LogManager.GetCurrentClassLogger();

            // Loading Config
            _logger.Info("Loading Configurations from App.config");
            LoadConfiguration();

            // Control Variable (Bool - Should the process use proxies? )
            bool shouldUseProxies = false;

            // Checking for the need to use proxies
            if (args != null && args.Length == 1)
            {
                // Setting flag to true
                shouldUseProxies = true;

                // Loading proxies from .txt received as argument
                String fPath = args[0];

                // Sanity Check
                if (!File.Exists(fPath))
                {
                    _logger.Fatal("Couldnt find proxies on path : " + fPath);
                    System.Environment.Exit(-100);
                }

                // Reading Proxies from File
                string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));

                try
                {
                    // Actual Load of Proxies
                    ProxiesLoader.Load(fLines.ToList());
                }
                catch (Exception ex)
                {
                    _logger.Fatal(ex);
                    System.Environment.Exit(-101);
                }
            }

            // AWS Queue Handler
            _logger.Info("Initializing Queues");
            AWSSQSHelper appsUrlQueue  = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
            AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);

            // Setting Error Flag to No Error ( 0 )
            System.Environment.ExitCode = 0;

            // Initialiazing Control Variables
            int fallbackWaitTime = 1;

            _logger.Info("Started Processing Individual Apps Urls");

            do
            {
                try
                {
                    // Dequeueing messages from the Queue
                    if (!appsUrlQueue.DeQueueMessages())
                    {
                        Thread.Sleep(_hiccupTime);  // Hiccup
                        continue;
                    }

                    // Checking for no message received, and false positives situations
                    if (!appsUrlQueue.AnyMessageReceived())
                    {
                        // If no message was found, increases the wait time
                        int waitTime;
                        if (fallbackWaitTime <= 12)
                        {
                            // Exponential increase on the wait time, truncated after 12 retries
                            waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
                        }
                        else // Reseting Wait after 12 fallbacks
                        {
                            waitTime         = 2000;
                            fallbackWaitTime = 0;
                        }

                        fallbackWaitTime++;

                        // Sleeping before next try
                        Console.WriteLine("Fallback (seconds) => " + waitTime);
                        Thread.Sleep(waitTime);
                        continue;
                    }

                    // Reseting fallback time
                    fallbackWaitTime = 1;

                    // Iterating over dequeued Messages
                    foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
                    {
                        if (appUrl.Body.IndexOf("https://itunes.apple.com/us/app") < 0 && appUrl.Body.IndexOf("https://itunes.apple.com/app/") < 0)
                        {
                            _logger.Info("Invalid Message. Deleting [{0}]", appUrl.Body);
                            appsUrlQueue.DeleteMessage(appUrl);
                            continue;
                        }

                        bool processingWorked = true;

                        try
                        {
                            // Retries Counter
                            int    retries = 0;
                            string htmlResponse;

                            // Retrying if necessary
                            do
                            {
                                // Executing Http Request for the Category Url
                                htmlResponse = httpClient.Get(appUrl.Body, shouldUseProxies);

                                if (String.IsNullOrEmpty(htmlResponse))
                                {
                                    // Extending Fallback time
                                    retries++;
                                    int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;

                                    _logger.Info("Retrying Request for App Page [ " + sleepTime / 1000 + " ]");

                                    Thread.Sleep(sleepTime);
                                }
                            } while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);

                            // Checking if retries failed
                            if (String.IsNullOrWhiteSpace(htmlResponse))
                            {
                                continue;
                            }

                            // Feedback
                            _logger.Info("Current page " + appUrl.Body, "Parsing App Data");

                            // Parsing Data out of the Html Page
                            AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
                            parsedApp.url = appUrl.Body;

                            // Enqueueing App Data
                            appsDataQueue.EnqueueMessage(parsedApp.ToJson());

                            // Little Hiccup
                            Thread.Sleep(_hiccupTime);
                        }
                        catch (Exception ex)
                        {
                            _logger.Error(ex);

                            // Setting Flag to "False"
                            processingWorked = false;
                        }
                        finally
                        {
                            //Deleting the message - Only if the processing worked
                            if (processingWorked)
                            {
                                appsUrlQueue.DeleteMessage(appUrl);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    _logger.Error(ex);
                }
            } while (true);
        }
Пример #3
0
        static void Main(string[] args)
        {
            // Loading Configuration
            LogSetup.InitializeLog("Apple_Store_Recorder.log", "info");
            _logger = LogManager.GetCurrentClassLogger();

            // Loading Config
            _logger.Info("Loading Configurations from App.config");
            LoadConfiguration();

            // Initializing Queue
            _logger.Info("Initializing Queue");
            AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
            AWSSQSHelper backup        = new AWSSQSHelper("DeadLetter", _maxMessagesPerDequeue, _awsKey, _awsKeySecret);

            // Creating MongoDB Instance
            _logger.Info("Loading MongoDB / Creating Instances");

            MongoDBWrapper mongoDB    = new MongoDBWrapper();
            string         serverAddr = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT);

            mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, serverAddr, 10000, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION);

            // Setting Error Flag to No Error ( 0 )
            System.Environment.ExitCode = 0;

            // Initialiazing Control Variables
            int fallbackWaitTime = 1;

            // Buffer of Messages to be recorder
            List <AppleStoreAppModel> recordsBuffer  = new List <AppleStoreAppModel> ();
            List <Message>            messagesBuffer = new List <Message> ();

            // Insert Batch Size
            int batchSize = 1000;

            _logger.Info("Started Recording App Data");

            do
            {
                try
                {
                    // Dequeueing messages from the Queue
                    if (!appsDataQueue.DeQueueMessages())
                    {
                        Thread.Sleep(_hiccupTime);  // Hiccup
                        continue;
                    }

                    // Checking for no message received, and false positives situations
                    if (!appsDataQueue.AnyMessageReceived())
                    {
                        // If no message was found, increases the wait time
                        int waitTime;
                        if (fallbackWaitTime <= 12)
                        {
                            // Exponential increase on the wait time, truncated after 12 retries
                            waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
                        }
                        else // Reseting Wait after 12 fallbacks
                        {
                            waitTime         = 2000;
                            fallbackWaitTime = 0;
                        }

                        fallbackWaitTime++;

                        // Sleeping before next try
                        Console.WriteLine("Fallback (seconds) => " + waitTime);
                        Thread.Sleep(waitTime);
                        continue;
                    }

                    // Reseting fallback time
                    fallbackWaitTime = 1;

                    // Iterating over dequeued Messages
                    foreach (var appDataMessage in appsDataQueue.GetDequeuedMessages())
                    {
                        try
                        {
                            // Deserializing message
                            var appData = AppleStoreAppModel.FromJson(appDataMessage.Body);

                            // Dumping "Url" to "_id"
                            appData._id = appData.url;

                            // Adding it to the buffer of records to be recorded
                            recordsBuffer.Add(appData);

                            // Adding message to the buffer of messages to be deleted
                            messagesBuffer.Add(appDataMessage);

                            // Is it time to batch insert ?
                            if ((recordsBuffer.Count % batchSize) == 0)
                            {
                                // Batch Insertion
                                mongoDB.BatchInsert <AppleStoreAppModel> (recordsBuffer);

                                // Logging Feedback
                                _logger.Info("\tApps Recorded : " + recordsBuffer.Count);

                                // Deleting Messages
                                messagesBuffer.ForEach((msg) => appsDataQueue.DeleteMessage(msg));

                                _logger.Info("\tMessages Deleted: " + messagesBuffer.Count);

                                // Clearing Buffers
                                recordsBuffer.Clear();
                                messagesBuffer.Clear();
                            }
                        }
                        catch (Exception ex)
                        {
                            _logger.Error(ex);
                        }
                        finally
                        {
                            // Deleting the message
                            appsDataQueue.DeleteMessage(appDataMessage);
                            backup.EnqueueMessage(appDataMessage.Body);
                        }
                    }
                }
                catch (Exception ex)
                {
                    _logger.Error(ex);
                }
            } while (true);
        }
Пример #4
0
        static void Main(string[] args)
        {
            // Creating Needed Instances
            RequestsHandler httpClient = new RequestsHandler();
            AppStoreParser  parser     = new AppStoreParser();

            _logger = new LogWrapper();

            // Loading Configuration
            _logger.LogMessage("Loading Configurations from App.config");
            LoadConfiguration();

            // AWS Queue Handler
            _logger.LogMessage("Initializing Queues");
            AWSSQSHelper appsUrlQueue  = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
            AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);

            // Setting Error Flag to No Error ( 0 )
            System.Environment.ExitCode = 0;

            // Initialiazing Control Variables
            int fallbackWaitTime = 1;

            _logger.LogMessage("Started Processing Individual Apps Urls");

            do
            {
                try
                {
                    // Dequeueing messages from the Queue
                    if (!appsUrlQueue.DeQueueMessages())
                    {
                        Thread.Sleep(_hiccupTime);  // Hiccup
                        continue;
                    }

                    // Checking for no message received, and false positives situations
                    if (!appsUrlQueue.AnyMessageReceived())
                    {
                        // If no message was found, increases the wait time
                        int waitTime;
                        if (fallbackWaitTime <= 12)
                        {
                            // Exponential increase on the wait time, truncated after 12 retries
                            waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
                        }
                        else // Reseting Wait after 12 fallbacks
                        {
                            waitTime         = 2000;
                            fallbackWaitTime = 0;
                        }

                        fallbackWaitTime++;

                        // Sleeping before next try
                        Console.WriteLine("Fallback (seconds) => " + waitTime);
                        Thread.Sleep(waitTime);
                        continue;
                    }

                    // Reseting fallback time
                    fallbackWaitTime = 1;

                    // Iterating over dequeued Messages
                    foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
                    {
                        try
                        {
                            // Retries Counter
                            int    retries = 0;
                            string htmlResponse;

                            // Retrying if necessary
                            do
                            {
                                // Executing Http Request for the Category Url
                                //appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8";
                                //appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8";
                                //appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8";
                                //appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8";
                                htmlResponse = httpClient.Get(appUrl.Body);

                                if (String.IsNullOrEmpty(htmlResponse))
                                {
                                    // Extending Fallback time
                                    retries++;
                                    int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;

                                    _logger.LogMessage("Retrying Request for App Page [ " + sleepTime / 1000 + " ]", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);

                                    Thread.Sleep(sleepTime);
                                }
                            } while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);

                            // Checking if retries failed
                            if (String.IsNullOrWhiteSpace(htmlResponse))
                            {
                                // Deletes Message and moves on
                                //appsUrlQueue.DeleteMessage (appUrl);
                                continue;
                            }

                            // Feedback
                            _logger.LogMessage("Current page " + appUrl.Body, "Parsing App Data");

                            // Parsing Data out of the Html Page
                            AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
                            parsedApp.url = appUrl.Body;

                            // Enqueueing App Data
                            appsDataQueue.EnqueueMessage(parsedApp.ToJson());

                            // Little Hiccup
                            Thread.Sleep(_hiccupTime);
                        }
                        catch (Exception ex)
                        {
                            _logger.LogMessage(ex.Message, "App Url Processing", BDC.BDCCommons.TLogEventLevel.Error);
                        }
                        finally
                        {
                            //Deleting the message
                            appsUrlQueue.DeleteMessage(appUrl);
                        }
                    }
                }
                catch (Exception ex)
                {
                    _logger.LogMessage(ex);
                }
            } while (true);
        }
Пример #5
0
        static void Main(string[] args)
        {
            // Creating needed Instances
            _logger = new LogWrapper();

            // Loading Configuration
            _logger.LogMessage("Loading Configurations from App.config");
            LoadConfiguration();

            // Initializing Queue
            _logger.LogMessage("Initializing Queue");
            AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);

            // Creating MongoDB Instance
            _logger.LogMessage("Loading MongoDB / Creating Instances");

            MongoDBWrapper mongoDB    = new MongoDBWrapper();
            string         serverAddr = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT);

            mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, serverAddr, 10000, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION);

            // Setting Error Flag to No Error ( 0 )
            System.Environment.ExitCode = 0;

            // Initialiazing Control Variables
            int fallbackWaitTime = 1;

            _logger.LogMessage("Started Processing App Urls");

            do
            {
                try
                {
                    // Dequeueing messages from the Queue
                    if (!appsDataQueue.DeQueueMessages())
                    {
                        Thread.Sleep(_hiccupTime);  // Hiccup
                        continue;
                    }

                    // Checking for no message received, and false positives situations
                    if (!appsDataQueue.AnyMessageReceived())
                    {
                        // If no message was found, increases the wait time
                        int waitTime;
                        if (fallbackWaitTime <= 12)
                        {
                            // Exponential increase on the wait time, truncated after 12 retries
                            waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
                        }
                        else // Reseting Wait after 12 fallbacks
                        {
                            waitTime         = 2000;
                            fallbackWaitTime = 0;
                        }

                        fallbackWaitTime++;

                        // Sleeping before next try
                        Console.WriteLine("Fallback (seconds) => " + waitTime);
                        Thread.Sleep(waitTime);
                        continue;
                    }

                    // Reseting fallback time
                    fallbackWaitTime = 1;

                    // Iterating over dequeued Messages
                    foreach (var appDataMessage in appsDataQueue.GetDequeuedMessages())
                    {
                        try
                        {
                            // Deserializing message
                            var appData = AppleStoreAppModel.FromJson(appDataMessage.Body);

                            // Checking for duplicates
                            if (!mongoDB.IsAppOnDatabase <AppleStoreAppModel> (appData.url))
                            {
                                // Recording App Data
                                mongoDB.Insert <AppleStoreAppModel> (appData);
                            }
                        }
                        catch (Exception ex)
                        {
                            _logger.LogMessage(ex.Message, "App Recording", BDC.BDCCommons.TLogEventLevel.Error);
                        }
                        finally
                        {
                            // Deleting the message
                            appsDataQueue.DeleteMessage(appDataMessage);
                        }
                    }
                }
                catch (Exception ex)
                {
                    _logger.LogMessage(ex);
                }
            } while (true);
        }