/// <summary> /// Entry point of the worker piece of the process /// Notice that you can run as many workers as you want to in order to make the crawling faster /// </summary> /// <param name="args"></param> static void Main(string[] args) { // Configuring Log Object Threshold LogWriter.Threshold = TLogEventLevel.Information; LogWriter.Info("Worker Started"); // Parser PlayStoreParser parser = new PlayStoreParser(); // Configuring MongoDB Wrapper MongoDBWrapper mongoDB = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); // Creating Instance of Web Requests Server WebRequests server = new WebRequests(); QueuedApp app; // Retry Counter (Used for exponential wait increasing logic) int retryCounter = 0; // Iterating Over MongoDB Records while no document is found to be processed while ((app = mongoDB.FindAndModify()) != null) { try { // Building APP URL string appUrl = Consts.APP_URL_PREFIX + app.Url; // Checking if this app is on the database already if (mongoDB.AppProcessed(appUrl)) { // Console Feedback, Comment this line to disable if you want to Console.WriteLine("Duplicated App, skipped."); // Delete it from the queue and continues the loop mongoDB.RemoveFromQueue(app.Url); continue; } // Configuring server and Issuing Request server.Headers.Add(Consts.ACCEPT_LANGUAGE); server.Host = Consts.HOST; server.Encoding = "utf-8"; server.EncodingDetection = WebRequests.CharsetDetection.DefaultCharset; string response = server.Get(appUrl); // Flag Indicating Success while processing and parsing this app bool ProcessingWorked = true; // Sanity Check if (String.IsNullOrEmpty(response) || server.StatusCode != System.Net.HttpStatusCode.OK) { LogWriter.Info("Error opening app page : " + appUrl); ProcessingWorked = false; // Renewing WebRequest Object to get rid of Cookies server = new WebRequests(); // Inc. retry counter retryCounter++; Console.WriteLine("Retrying:" + retryCounter); // Checking for maximum retry count double waitTime; if (retryCounter >= 11) { waitTime = TimeSpan.FromMinutes(35).TotalMilliseconds; // Removing App from the database (this the app page may have expired) mongoDB.RemoveFromQueue(app.Url); } else { // Calculating next wait time ( 2 ^ retryCounter seconds) waitTime = TimeSpan.FromSeconds(Math.Pow(2, retryCounter)).TotalMilliseconds; } // Hiccup to avoid google blocking connections in case of heavy traffic from the same IP Thread.Sleep(Convert.ToInt32(waitTime)); } else { // Reseting retry counter retryCounter = 0; // Parsing Useful App Data AppModel parsedApp = parser.ParseAppPage(response, appUrl); // Inserting App into Mongo DB Database if (!mongoDB.Insert <AppModel>(parsedApp)) { ProcessingWorked = false; } // If the processing failed, do not remove the app from the database, instead, keep it and flag it as not busy // so that other workers can try to process it later if (!ProcessingWorked) { mongoDB.ToggleBusyApp(app, false); } else // On the other hand, if processing worked, removes it from the database { // Console Feedback, Comment this line to disable if you want to Console.WriteLine("Inserted App : " + parsedApp.Name); mongoDB.RemoveFromQueue(app.Url); } // Counters for console feedback only int extraAppsCounter = 0, newExtraApps = 0; // Parsing "Related Apps" and "More From Developer" Apps (URLS Only) foreach (string extraAppUrl in parser.ParseExtraApps(response)) { // Incrementing counter of extra apps extraAppsCounter++; // Assembling Full app Url to check with database string fullExtraAppUrl = Consts.APP_URL_PREFIX + extraAppUrl; // Checking if the app was either processed or queued to be processed already if ((!mongoDB.AppProcessed(fullExtraAppUrl)) && (!mongoDB.IsAppOnQueue(extraAppUrl))) { // Incrementing counter of inserted apps newExtraApps++; // Adds it to the queue of apps to be processed mongoDB.AddToQueue(extraAppUrl); } } // Console Feedback Console.WriteLine("Queued " + newExtraApps + " / " + extraAppsCounter + " related apps"); } } catch (Exception ex) { LogWriter.Error(ex); } finally { try { // Toggles Busy status back to false mongoDB.ToggleBusyApp(app, false); } catch (Exception ex) { // Toggle Busy App may raise an exception in case of lack of internet connection, so, i must use this // "inner catch" to avoid it from happenning LogWriter.Error(ex); } } } }
/// <summary> /// Executes a Search using the searchField as the search parameter, /// paginates / scrolls the search results to the end adding all the url of apps /// it finds to a AWS SQS queue /// </summary> /// <param name="searchField"></param> private static void CrawlStore(string searchField) { // Console Feedback Console.WriteLine("Crawling Search Term : [ " + searchField + " ]"); // Compiling Regular Expression used to parse the "pagToken" out of the Play Store Regex pagTokenRegex = new Regex(@"GAEi+.+\:S\:.{11}\\42", RegexOptions.Compiled); // HTML Response string response; // MongoDB Helper // Configuring MongoDB Wrapper MongoDBWrapper mongoDB = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); // Ensuring the database has the proper indexe mongoDB.EnsureIndex("Url"); // Response Parser PlayStoreParser parser = new PlayStoreParser(); // Executing Web Requests using (WebRequests server = new WebRequests()) { // Creating Request Object server.Host = Consts.HOST; // Executing Initial Request response = server.Post(String.Format(Consts.CRAWL_URL, searchField), Consts.INITIAL_POST_DATA); // Parsing Links out of Html Page (Initial Request) foreach (string url in parser.ParseAppUrls(response)) { // Checks whether the app have been already processed // or is queued to be processed if ((!mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!mongoDB.AppQueued(url))) { // Console Feedback Console.WriteLine(" . Queued App"); // Than, queue it :) mongoDB.AddToQueue(url); Thread.Sleep(250); // Hiccup } else { // Console Feedback Console.WriteLine(" . Duplicated App. Skipped"); } } // Executing Requests for more Play Store Links int initialSkip = 48; int currentMultiplier = 1; int errorsCount = 0; do { // Finding pagToken from HTML var rgxMatch = pagTokenRegex.Match(response); // If there's no match, skips it if (!rgxMatch.Success) { break; } // Reading Match from Regex, and applying needed replacements string pagToken = rgxMatch.Value.Replace(":S:", "%3AS%3A").Replace("\\42", String.Empty).Replace(@"\\u003d", String.Empty); // Assembling new PostData with paging values string postData = String.Format(Consts.POST_DATA, pagToken); // Executing request for values response = server.Post(String.Format(Consts.CRAWL_URL, searchField), postData); // Checking Server Status if (server.StatusCode != System.Net.HttpStatusCode.OK) { LogWriter.Error("Http Error", "Status Code [ " + server.StatusCode + " ]"); errorsCount++; continue; } // Parsing Links foreach (string url in parser.ParseAppUrls(response)) { // Checks whether the app have been already processed // or is queued to be processed if ((!mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!mongoDB.AppQueued(url))) { // Console Feedback Console.WriteLine(" . Queued App"); // Than, queue it :) mongoDB.AddToQueue(url); Thread.Sleep(250); // Hiccup } else { // Console Feedback Console.WriteLine(" . Duplicated App. Skipped"); } } // Incrementing Paging Multiplier currentMultiplier++; } while (parser.AnyResultFound(response) && errorsCount <= Consts.MAX_REQUEST_ERRORS); } }
static void Main(string[] args) { // Configuring Log Object Logger logger = LogManager.GetCurrentClassLogger(); // Control Variable (Bool - Should the process use proxies? ) bool isUsingProxies = false; // Checking for the need to use proxies if (args != null && args.Length == 1) { // Setting flag to true isUsingProxies = true; // Loading proxies from .txt received as argument String fPath = args[0]; // Sanity Check if (!File.Exists(fPath)) { logger.Fatal("Couldnt find proxies on path : " + fPath); System.Environment.Exit(-100); } // Reading Proxies from File string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8")); try { // Actual Load of Proxies ProxiesLoader.Load(fLines.ToList()); } catch (Exception ex) { logger.Fatal(ex); System.Environment.Exit(-101); } } // MongoDB instance Creation logger.Info("Configuring MonboDB Client"); // Creating instance of Mongo Handler for the main collection MongoDBWrapper mongoClient = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoClient.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); logger.Info("Iterating over Apps"); // Creating Play Store Parser PlayStoreParser parser = new PlayStoreParser(); // App Model AppModel appRecord; // Control Variable bool noError = true; // Finding all the "Apps" that didn't have the reviews visited yet while ((appRecord = mongoClient.FindAndModifyReviews()) != null) { // Extracting app ID from URL string appId = appRecord.Url.Replace(Consts.PLAY_STORE_PREFIX, String.Empty); // Console Feedback logger.Info("Processing App [ " + appRecord.Name + " ] "); try { // Console Feedback Console.Write("Reviews from : " + appRecord.Name); // Issuing Request for Reviews string response = ReviewsWrapper.GetAppReviews(appId, 1, isUsingProxies); // Checking for Blocking Situation if (String.IsNullOrEmpty(response)) { logger.Info("Blocked by Play Store. Sleeping process for 10 minutes before retrying."); // Thread Wait for 10 seconds Thread.Sleep(TimeSpan.FromSeconds(10)); } // Checking for "No Reviews" app if (response.Length < 50) { logger.Info("No Reviews for this app. Skipping"); Console.Write(" - No Reviews Found\n"); continue; } // Normalizing Response to Proper HTML response = ReviewsWrapper.NormalizeResponse(response); // List of Reviews List <AppReview> reviews = new List <AppReview> (); // Iterating over Parsed Reviews foreach (var review in parser.ParseReviews(response)) { // Adding App Data to the review review.appID = appId; review.appName = appRecord.Name; review.appURL = appRecord.Url; // Capture Timestamp to the model review.timestamp = DateTime.Now; // Adding reviews to the current list reviews.Add(review); } // Any Review Found ? if (reviews.Count > 0) { Console.Write(" - " + reviews.Count + " Reviews Found\n"); // Checking if there was any previous list of reviews if (appRecord.Reviews == null) { appRecord.Reviews = reviews; } else // Previous List found - Appending only the new ones { foreach (var review in reviews) { if (!appRecord.Reviews.Any(t => t.permalink.Equals(review.permalink))) { appRecord.Reviews.Add(review); } } } } } catch (Exception ex) { logger.Error(ex); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Error : " + ex.Message); Console.ForegroundColor = ConsoleColor.White; noError = false; } finally { // Toggling back the "ReviewsStatus" attribute from the model if (noError) { appRecord.ReviewsStatus = "Visited"; mongoClient.SaveRecord <AppModel> (appRecord); } else // "Error" status { appRecord.ReviewsStatus = "Error"; mongoClient.SaveRecord <AppModel> (appRecord); } } } }
/// <summary> /// Entry point of the worker piece of the process /// Notice that you can run as many workers as you want to in order to make the crawling faster /// </summary> /// <param name="args"></param> static void Main(string[] args) { // Configuring Log Object LogSetup.InitializeLog("PlayStoreWorker.log", "info"); Logger logger = LogManager.GetCurrentClassLogger(); logger.Info("Worker Started"); // Control Variable (Bool - Should the process use proxies? ) bool isUsingProxies = false; // Checking for the need to use proxies if (args != null && args.Length == 1) { // Setting flag to true isUsingProxies = true; // Loading proxies from .txt received as argument String fPath = args[0]; // Sanity Check if (!File.Exists(fPath)) { logger.Fatal("Couldnt find proxies on path : " + fPath); System.Environment.Exit(-100); } // Reading Proxies from File string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8")); try { // Actual Load of Proxies ProxiesLoader.Load(fLines.ToList()); } catch (Exception ex) { logger.Fatal(ex); System.Environment.Exit(-101); } } // Parser PlayStoreParser parser = new PlayStoreParser(); // Configuring MongoDB Wrapper MongoDBWrapper mongoDB = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); /* * // populate min downloaded & max downloaded * int count = 0; * var apps = mongoDB.FindAll<AppModel>(); * foreach(var a in apps) * { * a.FillMinAndMaxInstalls(); ++count; * * if((count % 100) == 0) * { * Console.WriteLine("updated {0}", count); * } * * if (!mongoDB.UpsertKeyEq<AppModel>(a, "Url", a.Url)) * { * Console.WriteLine("UpsertKey failed"); * } * } */ // Creating Instance of Web Requests Server WebRequests server = new WebRequests(); // Queued App Model QueuedApp app; // Retry Counter (Used for exponential wait increasing logic) int retryCounter = 0; // Iterating Over MongoDB Records while no document is found to be processed while ((app = mongoDB.FindAndModify()) != null) { try { // Building APP URL string appUrl = app.Url; // Sanity check of app page url if (app.Url.IndexOf("http", StringComparison.OrdinalIgnoreCase) < 0) { appUrl = Consts.APP_URL_PREFIX + app.Url; } // Checking if this app is on the database already if (mongoDB.AppProcessed(appUrl)) { // Console Feedback, Comment this line to disable if you want to logger.Info("Duplicated App, skipped."); // Delete it from the queue and continues the loop mongoDB.RemoveFromQueue(app.Url); continue; } // Configuring server and Issuing Request server.Headers.Add(Consts.ACCEPT_LANGUAGE); server.Host = Consts.HOST; server.UserAgent = Consts.GITHUBURL; server.Encoding = "utf-8"; server.EncodingDetection = WebRequests.CharsetDetection.DefaultCharset; // Checking for the need to use "HTTP Proxies" if (isUsingProxies) { server.Proxy = ProxiesLoader.GetWebProxy(); } // Issuing HTTP Request string response = server.Get(appUrl); // Flag Indicating Success while processing and parsing this app bool ProcessingWorked = true; // Sanity Check if (String.IsNullOrEmpty(response) || server.StatusCode != System.Net.HttpStatusCode.OK) { logger.Info("Error opening app page : " + appUrl); ProcessingWorked = false; if (isUsingProxies) { ProxiesLoader.IncrementCurrentProxy(); } // Renewing WebRequest Object to get rid of Cookies server = new WebRequests(); // Fallback time variable double waitTime; // Checking which "Waiting Logic" to use - If there are proxies being used, there's no need to wait too much // If there are no proxies in use, on the other hand, the process must wait more time if (isUsingProxies) { // Waits two seconds everytime waitTime = TimeSpan.FromSeconds(2).TotalMilliseconds; } else { // Increments retry counter retryCounter++; // Checking for maximum retry count if (retryCounter >= 8) { waitTime = TimeSpan.FromMinutes(20).TotalMilliseconds; } else { // Calculating next wait time ( 2 ^ retryCounter seconds) waitTime = TimeSpan.FromSeconds(Math.Pow(2, retryCounter)).TotalMilliseconds; } } // Hiccup to avoid google blocking connections in case of heavy traffic from the same IP logger.Info("======================================================"); logger.Info("\n\tFallback : " + waitTime + " Seconds"); Thread.Sleep(Convert.ToInt32(waitTime)); // If The Status code is "ZERO" (it means 404) - App must be removed from "Queue" if (server.StatusCode == 0) { // Console Feedback logger.Info("\tApp Not Found (404) - " + app.Url); mongoDB.RemoveFromQueue(app.Url); } logger.Info("======================================================"); } else { // Reseting retry counter retryCounter = 0; // Parsing Useful App Data AppModel parsedApp = parser.ParseAppPage(response, appUrl); // Normalizing URLs if (!String.IsNullOrWhiteSpace(parsedApp.DeveloperPrivacyPolicy)) { parsedApp.DeveloperPrivacyPolicy = parsedApp.DeveloperPrivacyPolicy.Replace("https://www.google.com/url?q=", String.Empty); } if (!String.IsNullOrWhiteSpace(parsedApp.DeveloperWebsite)) { parsedApp.DeveloperNormalizedDomain = parser.NormalizeDomainName(parsedApp.DeveloperWebsite); } List <String> relatedApps = new List <String> (); // Avoiding Exceptions caused by "No Related Apps" situations - Must be treated differently try { // Parsing "Related Apps" and "More From Developer" Apps (URLS Only) foreach (string extraAppUrl in parser.ParseExtraApps(response)) { relatedApps.Add(Consts.APP_URL_PREFIX + extraAppUrl); } // Adding "Related Apps" to Apps Model parsedApp.RelatedUrls = relatedApps.Distinct().ToArray(); } catch { logger.Info("\tNo Related Apps Found. Skipping"); } // Inserting App into Mongo DB Database if (!mongoDB.UpsertKeyEq <AppModel>(parsedApp, "Url", appUrl)) { ProcessingWorked = false; } // If the processing failed, do not remove the app from the database, instead, keep it and flag it as not busy // so that other workers can try to process it later if (!ProcessingWorked) { mongoDB.ToggleBusyApp(app, false); } else // On the other hand, if processing worked, removes it from the database { // Console Feedback, Comment this line to disable if you want to Console.ForegroundColor = ConsoleColor.Red; logger.Info("Inserted App : " + parsedApp.Name); Console.ForegroundColor = ConsoleColor.White; mongoDB.RemoveFromQueue(app.Url); } // Counters for console feedback only int extraAppsCounter = 0, newExtraApps = 0; // Parsing "Related Apps" and "More From Developer" Apps (URLS Only) foreach (string extraAppUrl in relatedApps) { // Incrementing counter of extra apps extraAppsCounter++; // Assembling Full app Url to check with database string fullExtraAppUrl; if (extraAppUrl.IndexOf("https://play.google.com/") >= 0) { fullExtraAppUrl = extraAppUrl; } else { fullExtraAppUrl = Consts.APP_URL_PREFIX + extraAppUrl; } // Checking if the app was either processed or queued to be processed already if ((!mongoDB.AppProcessed(fullExtraAppUrl)) && (!mongoDB.IsAppOnQueue(extraAppUrl))) { // Incrementing counter of inserted apps newExtraApps++; // Adds it to the queue of apps to be processed mongoDB.AddToQueue(extraAppUrl); } } // Console Feedback logger.Info("Queued " + newExtraApps + " / " + extraAppsCounter + " related apps"); } } catch (Exception ex) { logger.Error(ex); } finally { try { // Toggles Busy status back to false mongoDB.ToggleBusyApp(app, false); } catch (Exception ex) { // Toggle Busy App may raise an exception in case of lack of internet connection, so, i must use this // "inner catch" to avoid it from happenning logger.Error(ex); } } } }
static void Main(string[] args) { // Checking for Input Parameters if (args == null || args.Length != 1) { Console.WriteLine("Incorrect number of arguments received. Expected One"); System.Environment.Exit(-100); } // Human Readable Variable string inputFile = args[0]; // Checking if the Input file received exists if (!File.Exists(inputFile)) { Console.WriteLine(String.Format("Received input file does not exist : {0}", inputFile)); System.Environment.Exit(-101); } // App Status _appStatus = new Dictionary <String, AppStatusModel> (); // Creating Instance of Database Manager MongoDBWrapper mongoDB = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); // Creating Instance of Parser PlayStoreParser dataParser = new PlayStoreParser(); goto PeopleData; using (WebRequests httpClient = new WebRequests()) { // Minor Configuration of the Http Client - Ensures that the requests response will be in english // By doing so, we have no problems parsing the dates to their proper formats httpClient.Headers.Add(Consts.ACCEPT_LANGUAGE); httpClient.Host = Consts.HOST; httpClient.Encoding = "utf-8"; httpClient.EncodingDetection = WebRequests.CharsetDetection.DefaultCharset; // Iterating over File Lines (App Urls) - To Extract Data, Not The Reviews Yet. foreach (string appUrl in File.ReadAllLines(inputFile)) { // Logging Progress Console.WriteLine("\n => Processing App : " + appUrl); // Executing Http Get Request for the Apps's Data - With max of 5 Retries String appDataResponse = String.Empty; int currentRetry = 0; do { // Http Get appDataResponse = httpClient.Get(appUrl); } while (String.IsNullOrWhiteSpace(appDataResponse) || ++currentRetry <= _maxRetries); // Sanity Check if (String.IsNullOrWhiteSpace(appDataResponse)) { Console.WriteLine("\t\t.Error - Failed to find page of app : " + appUrl + ". Skipping it"); continue; } Console.WriteLine("\t\t.Page Found. Firing Parser"); // Parsing App Data AppModel appData = dataParser.ParseAppPage(appDataResponse, appUrl); // Checking If this app is on the database already if (mongoDB.AppProcessed(appUrl)) { Console.WriteLine("\t\t.Previous Version of App Found. Updating It"); mongoDB.UpdateRecord(appData, "Url", appData.Url); // Updating App Status _appStatus.Add ( appData.Url, new AppStatusModel() { appId = appData.Url.Replace(Consts.PLAY_STORE_PREFIX, String.Empty), appUrl = appData.Url, appName = appData.Name, status = "Updated" } ); } else { Console.WriteLine("\t\t.No Previous Version of the App Found. Adding to Database"); mongoDB.Insert <AppModel> (appData); // Updating App Status _appStatus.Add ( appData.Url, new AppStatusModel() { appId = appData.Url.Replace(Consts.PLAY_STORE_PREFIX, String.Empty), appUrl = appData.Url, appName = appData.Name, status = "Inserted" } ); } } } Reviews: // Next Phase: Parse Reviews of those Apps Console.WriteLine("\n => Parsing Complete. Obtaining Reviews"); // Iterating again over app urls to parse the reviews from this app foreach (string appUrl in File.ReadAllLines(inputFile)) { // Reaching App Id string appID = _appStatus[appUrl].appId; // Reviews-Break-Parsing Flag bool shouldContinueParsing = true; // Parsing Review Pages from the apps for (int currentPage = 1; /* no stop condition */; currentPage++) { // Getting Reviews Data Bundle string reviewsData = ReviewsWrapper.GetAppReviews(appID, currentPage); // Checking for Blocking Situation if (String.IsNullOrEmpty(reviewsData)) { Console.WriteLine("Blocked by Play Store. Sleeping process for 10 minutes before retrying."); // Thread Wait for 10 Minutes Thread.Sleep(10 * 60 * 1000); } // Checking for "No Reviews" app if (reviewsData.Length < 50) { Console.WriteLine("No Reviews left for this app. Skipping"); break; } // Normalizing Response to Proper HTML reviewsData = ReviewsWrapper.NormalizeResponse(reviewsData); // Iterating over Parsed Reviews foreach (var review in dataParser.ParseReviews(reviewsData)) { // Adding App Data to the review review.appID = _appStatus[appUrl].appId; review.appName = _appStatus[appUrl].appName; review.appURL = _appStatus[appUrl].appUrl; // Incrementing Reviews Count for this app _appStatus[appUrl].reviews++; // Adding Review Object to Database review.timestamp = DateTime.Now; // Building Query to check for duplicated review var duplicatedReviewQuery = Query.EQ("permalink", review.permalink); // Checking for duplicated review before inserting it if (mongoDB.FindMatch <AppReview> (duplicatedReviewQuery, 1, 0, Consts.REVIEWS_COLLECTION).Count() == 0) { // Inserting Review into MongoDB mongoDB.Insert <AppReview> (review, Consts.REVIEWS_COLLECTION); } else { Console.WriteLine("Duplicated Review. Skipping App"); // When this happens, there are no more reviews to be parsed shouldContinueParsing = false; // Skipping this apps processing } } // Hiccup to avoid Blocking problems Console.WriteLine("Parsed Reviews: " + _appStatus[appUrl].reviews); Thread.Sleep(new Random().Next(14000, 21000)); if (!shouldContinueParsing) { break; } } } PeopleData: Console.WriteLine("\n\n => Processing People Data"); Console.WriteLine("\nSimulating Google Login Using Selenium."); using (var firefoxDriver = new FirefoxDriver()) { // Navigating to Dummy Url - One that I Know that well be asked for a login firefoxDriver.Navigate().GoToUrl("https://play.google.com/store/people/details?id=101242565951396343093"); // Reaching Login Fields var loginField = firefoxDriver.FindElementById("Email"); var passwordField = firefoxDriver.FindElementById("Passwd"); var btnSignIn = firefoxDriver.FindElementById("signIn"); // Sending Credentials to the browser loginField.SendKeys("YOUREMAIL"); passwordField.SendKeys("YOURPASSWORD"); btnSignIn.Click(); string lastPeople = "https://play.google.com/store/people/details?id=115037241907660526856"; bool shouldcontinue = false; // Processing Reviewers Data foreach (string peopleUrl in mongoDB.FindPeopleUrls()) { // Skipping until last link if (peopleUrl == lastPeople) { shouldcontinue = true; } if (!shouldcontinue) { continue; } // Navigating To the Reviewer Page firefoxDriver.Navigate().GoToUrl(peopleUrl); // Executing Get Request for the Reviewer page on Google Play string reviewerPage = firefoxDriver.PageSource; // Extracting Reviewer Data from the Page ReviewerPageData reviewerData = dataParser.ParsePeopleData(reviewerPage); // Adding Url to the model reviewerData.reviewerUrl = peopleUrl; // Inserting it to the database - If no previous record of this Reviewer is found if (!mongoDB.IsReviewerOnDatabase(peopleUrl)) { mongoDB.Insert <ReviewerPageData> (reviewerData, "ReviewersData"); } } } // End of Processing + Console Feedback Console.WriteLine("\n\n == Processing Summary =="); foreach (var status in _appStatus.Select(t => t.Value)) { // Message string cMessage = "=> App : {0} - Status {1} - Reviews : {2}"; Console.WriteLine(String.Format(cMessage, status.appName, status.status, status.reviews)); } Console.ReadLine(); }
static void Main(string[] args) { // Configuring Log Object Logger logger = LogManager.GetCurrentClassLogger(); // Parsing Arguments logger.Info("Checking for Arguments"); if (args == null || args.Length != 3) { logger.Fatal("Arguments Fatal", "Incorrect number of arguments received. Try passing two."); return; // Halts. } logger.Info("Reading Arguments"); // Reading actual arguments received _arguments.Add("AppsToProcess", Int32.Parse(args[0])); _arguments.Add("ReviewsPagePerApp", Int32.Parse(args[1])); _arguments.Add("AppsToSkip", Int32.Parse(args[2])); // Building MongoDB Query - This query specifies which applications you want to parse out the reviews // For more regarding MongoDB Queries, check the documentation on the project wiki page //var mongoQuery = Query.EQ ("Instalations", "1,000,000 - 5,000,000"); var mongoQuery = Query.EQ("Category", "/store/apps/category/EDUCATION"); logger.Info("Configuring MonboDB Client"); // Creating instance of Mongo Handler for the main collection MongoDBWrapper mongoClient = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoClient.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); logger.Info("Iterating over Apps"); // Creating Play Store Parser PlayStoreParser parser = new PlayStoreParser(); // Iterating over Query Results for the App Ids foreach (var appRecord in mongoClient.FindMatch <AppModel>(mongoQuery, _arguments["AppsToProcess"], _arguments["AppsToSkip"])) { // Extracting app ID from URL string appId = appRecord.Url.Replace(Consts.PLAY_STORE_PREFIX, String.Empty); // Console Feedback logger.Info("Processing App [ " + appRecord.Name + " ] "); bool shouldSkipApp = false; // Iterating over Review Pages up to the max received as argument for (int currentPage = 1; currentPage <= _arguments["ReviewsPagePerApp"]; currentPage++) { // Checking for the need to skip this app in case of duplicated review if (shouldSkipApp) { break; } try { // Page Feedback logger.Info("\tCurrent Page: " + currentPage); // Issuing Request for Reviews string response = ReviewsWrapper.GetAppReviews(appId, currentPage); // Checking for Blocking Situation if (String.IsNullOrEmpty(response)) { logger.Info("Blocked by Play Store. Sleeping process for 10 minutes before retrying."); // Thread Wait for 10 Minutes Thread.Sleep(10 * 60 * 1000); } // Checking for "No Reviews" app if (response.Length < 50) { logger.Info("No Reviews for this app. Skipping"); break; } // Normalizing Response to Proper HTML response = ReviewsWrapper.NormalizeResponse(response); // Iterating over Parsed Reviews foreach (var review in parser.ParseReviews(response)) { // Adding App Data to the review review.appID = appId; review.appName = appRecord.Name; review.appURL = appRecord.Url; // Adding processing timestamp to the model review.timestamp = DateTime.Now; // Building Query to check for duplicated review var duplicatedReviewQuery = Query.EQ("permalink", review.permalink); // Checking for duplicated review before inserting it if (mongoClient.FindMatch <AppReview>(duplicatedReviewQuery, 1, 0, Consts.REVIEWS_COLLECTION).Count() == 0) { // Inserting Review into MongoDB mongoClient.Insert <AppReview>(review, Consts.REVIEWS_COLLECTION); } else { logger.Info("Duplicated Review", "Review already parsed. Skipping App"); //shouldSkipApp = true; //break; } } } catch (Exception ex) { logger.Error(ex); } } } }
/// <summary> /// /// </summary> /// <param name="categoryUrl"></param> private static void CrawlCategory(string categoryUrl, string categoryName, bool shouldUseProxies) { // Console Feedback _logger.Warn("Crawling Category : [ " + categoryName + " ]"); // Hashset of urls used to keep track of what's been parsed already HashSet <String> foundUrls = new HashSet <String> (); // Control variable to avoid "Loop" on pagging bool isDonePagging = false; // HTML Response string response; // MongoDB Helper // Configuring MongoDB Wrapper MongoDBWrapper mongoDB = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); // Ensuring the database has the proper indexe mongoDB.EnsureIndex("Url"); // Response Parser PlayStoreParser parser = new PlayStoreParser(); // Executing Web Requests using (WebRequests server = new WebRequests()) { // Creating Request Object server.Headers.Add(Consts.ACCEPT_LANGUAGE); server.Host = Consts.HOST; server.UserAgent = Consts.GITHUBURL; server.Encoding = "utf-8"; // Executing Initial Request response = server.Get(categoryUrl); // Parsing Links out of Html Page (Initial Request) foreach (string url in parser.ParseAppUrls(response)) { // Saving found url on local hashset foundUrls.Add(url); // Checks whether the app have been already processed // or is queued to be processed if ((!mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!mongoDB.AppQueued(url))) { // Than, queue it :) mongoDB.AddToQueue(url); } } // Executing Requests for more Play Store Links int baseSkip = 60; int currentMultiplier = 1; int errorsCount = 0; do { // Assembling new PostData with paging values string postData = String.Format(Consts.CATEGORIES_POST_DATA, (currentMultiplier * baseSkip), baseSkip); // Executing request for values response = server.Post(String.Format(categoryUrl + "?authuser=0"), postData); // Checking Server Status if (server.StatusCode != System.Net.HttpStatusCode.OK) { _logger.Error("Http Error" + " - Status Code [ " + server.StatusCode + " ]"); errorsCount++; continue; } // Parsing Links foreach (string url in parser.ParseAppUrls(response)) { // If a certain app is found twice, it means that the "pagging" logic got stuck into a // Loop, so the all the apps for this category were parsed already if (foundUrls.Contains(url)) { isDonePagging = true; break; } // Saving found url on local hashset foundUrls.Add(url); // Checks whether the app have been already processed // or is queued to be processed if ((!mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!mongoDB.AppQueued(url))) { // Than, queue it :) mongoDB.AddToQueue(url); } } // Incrementing Paging Multiplier currentMultiplier++; } while (!isDonePagging && errorsCount <= Consts.MAX_REQUEST_ERRORS); } }
/// <summary> /// Executes a Search using the searchField as the search parameter, /// paginates / scrolls the search results to the end adding all the url of apps /// it finds to a Mongo "QUEUE" collection /// </summary> /// <param name="searchField"></param> private static void CrawlStore(string searchField, bool shouldUseProxies) { // Console Feedback _logger.Warn("Crawling Search Term : [ " + searchField + " ]"); // Hashset of urls used to keep track of what's been parsed already HashSet <String> foundUrls = new HashSet <String> (); // Control variable to avoid "Loop" on pagging bool isDonePagging = false; // Compiling Regular Expression used to parse the "pagToken" out of the Play Store Regex pagTokenRegex = new Regex(@"GAEi+.+\:S\:.{11}\\x22", RegexOptions.Compiled); // HTML Response string response; // Response Parser PlayStoreParser parser = new PlayStoreParser(); // Executing Web Requests using (WebRequests server = new WebRequests()) { // Creating Request Object server.Headers.Add(Consts.ACCEPT_LANGUAGE); server.Host = Consts.HOST; server.UserAgent = Consts.GITHUBURL; server.Encoding = "utf-8"; // Checking for the need to use "HTTP Proxies" if (shouldUseProxies) { server.Proxy = ProxiesLoader.GetWebProxy(); } // Executing Initial Request response = server.Post(String.Format(Consts.CRAWL_URL, searchField), Consts.INITIAL_POST_DATA); // Parsing Links out of Html Page (Initial Request) foreach (string url in parser.ParseAppUrls(response)) { // Checks whether the app have been already processed // or is queued to be processed foundUrls.Add(url); if ((!_mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!_mongoDB.AppQueued(url))) { // Than, queue it :) _mongoDB.AddToQueue(url); Thread.Sleep(250); // Hiccup } } // Executing Requests for more Play Store Links int initialSkip = 48; int currentMultiplier = 1; int errorsCount = 0; do { // Finding pagToken from HTML var rgxMatch = pagTokenRegex.Match(response); // If there's no match, skips it if (!rgxMatch.Success) { break; } // Reading Match from Regex, and applying needed replacements string pagToken = rgxMatch.Value.Replace(":S:", "%3AS%3A").Replace("\\x22", String.Empty).Replace(@"\\u003d", String.Empty); // Assembling new PostData with paging values string postData = String.Format(Consts.POST_DATA, pagToken); // Executing request for values response = server.Post(String.Format(Consts.CRAWL_URL, searchField), postData); // Checking Server Status if (server.StatusCode != System.Net.HttpStatusCode.OK) { _logger.Error("Http Error" + " - Status Code [ " + server.StatusCode + " ]"); errorsCount++; continue; } // Parsing Links foreach (string url in parser.ParseAppUrls(response)) { if (foundUrls.Contains(url)) { isDonePagging = true; break; } // Checks whether the app have been already processed foundUrls.Add(url); if ((!_mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!_mongoDB.AppQueued(url))) { // Than, queue it :) _mongoDB.AddToQueue(url); Thread.Sleep(250); // Hiccup } } // Incrementing Paging Multiplier currentMultiplier++; } while (!isDonePagging && parser.AnyResultFound(response) && errorsCount <= Consts.MAX_REQUEST_ERRORS); } }
/// <summary> /// Executes a Search using the searchField as the search parameter, /// paginates / scrolls the search results to the end adding all the url of apps /// it finds to a AWS SQS queue /// </summary> /// <param name="searchField"></param> private static void CrawlStore(string searchField) { // Console Feedback Console.WriteLine("Crawling Search Term : [ " + searchField + " ]"); // HTML Response string response; // MongoDB Helper // Configuring MongoDB Wrapper MongoDBWrapper mongoDB = new MongoDBWrapper(); string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT); mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION); // Response Parser PlayStoreParser parser = new PlayStoreParser(); // Executing Web Requests using (WebRequests server = new WebRequests()) { // Creating Request Object server.Host = Consts.HOST; // Executing Initial Request response = server.Post(Consts.CRAWL_URL, Consts.INITIAL_POST_DATA); // Parsing Links out of Html Page (Initial Request) foreach (string url in parser.ParseAppUrls(response)) { // Checks whether the app have been already processed // or is queued to be processed if ((!mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!mongoDB.AppQueued(url))) { // Console Feedback Console.WriteLine(" . Queued App"); // Than, queue it :) mongoDB.AddToQueue(url); } else { // Console Feedback Console.WriteLine(" . Duplicated App. Skipped"); } } // Executing Requests for more Play Store Links int initialSkip = 48; int currentMultiplier = 1; int errorsCount = 0; do { // Assembling new PostData with paging values string postData = String.Format(Consts.POST_DATA, (initialSkip * currentMultiplier)); // Executing request for values response = server.Post(Consts.CRAWL_URL, postData); // Checking Server Status if (server.StatusCode != System.Net.HttpStatusCode.OK) { LogWriter.Error("Http Error", "Status Code [ " + server.StatusCode + " ]"); errorsCount++; continue; } // Parsing Links foreach (string url in parser.ParseAppUrls(response)) { // Checks whether the app have been already processed // or is queued to be processed if ((!mongoDB.AppProcessed(Consts.APP_URL_PREFIX + url)) && (!mongoDB.AppQueued(url))) { // Console Feedback Console.WriteLine(" . Queued App"); // Than, queue it :) mongoDB.AddToQueue(url); } else { // Console Feedback Console.WriteLine(" . Duplicated App. Skipped"); } } // Incrementing Paging Multiplier currentMultiplier++; } while (parser.AnyResultFound(response) && errorsCount <= Consts.MAX_REQUEST_ERRORS); } }