static void Main(string[] args)
{
// Configuring Log Object
Logger logger = LogManager.GetCurrentClassLogger();
// Control Variable (Bool - Should the process use proxies? )
bool isUsingProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
isUsingProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// MongoDB instance Creation
logger.Info("Configuring MonboDB Client");
// Creating instance of Mongo Handler for the main collection
MongoDBWrapper mongoClient = new MongoDBWrapper();
string fullServerAddress = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT);
mongoClient.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION);
logger.Info("Iterating over Apps");
// Creating Play Store Parser
PlayStoreParser parser = new PlayStoreParser();
// App Model
AppModel appRecord;
// Control Variable
bool noError = true;
// Finding all the "Apps" that didn't have the reviews visited yet
while ((appRecord = mongoClient.FindAndModifyReviews()) != null)
{
// Extracting app ID from URL
string appId = appRecord.Url.Replace(Consts.PLAY_STORE_PREFIX, String.Empty);
// Console Feedback
logger.Info("Processing App [ " + appRecord.Name + " ] ");
try
{
// Console Feedback
Console.Write("Reviews from : " + appRecord.Name);
// Issuing Request for Reviews
string response = ReviewsWrapper.GetAppReviews(appId, 1, isUsingProxies);
// Checking for Blocking Situation
if (String.IsNullOrEmpty(response))
{
logger.Info("Blocked by Play Store. Sleeping process for 10 minutes before retrying.");
// Thread Wait for 10 seconds
Thread.Sleep(TimeSpan.FromSeconds(10));
}
// Checking for "No Reviews" app
if (response.Length < 50)
{
logger.Info("No Reviews for this app. Skipping");
Console.Write(" - No Reviews Found\n");
continue;
}
// Normalizing Response to Proper HTML
response = ReviewsWrapper.NormalizeResponse(response);
// List of Reviews
List <AppReview> reviews = new List <AppReview> ();
// Iterating over Parsed Reviews
foreach (var review in parser.ParseReviews(response))
{
// Adding App Data to the review
review.appID = appId;
review.appName = appRecord.Name;
review.appURL = appRecord.Url;
// Capture Timestamp to the model
review.timestamp = DateTime.Now;
// Adding reviews to the current list
reviews.Add(review);
}
// Any Review Found ?
if (reviews.Count > 0)
{
Console.Write(" - " + reviews.Count + " Reviews Found\n");
// Checking if there was any previous list of reviews
if (appRecord.Reviews == null)
{
appRecord.Reviews = reviews;
}
else // Previous List found - Appending only the new ones
{
foreach (var review in reviews)
{
if (!appRecord.Reviews.Any(t => t.permalink.Equals(review.permalink)))
{
appRecord.Reviews.Add(review);
}
}
}
}
}
catch (Exception ex)
{
logger.Error(ex);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Error : " + ex.Message);
Console.ForegroundColor = ConsoleColor.White;
noError = false;
}
finally
{
// Toggling back the "ReviewsStatus" attribute from the model
if (noError)
{
appRecord.ReviewsStatus = "Visited";
mongoClient.SaveRecord <AppModel> (appRecord);
}
else // "Error" status
{
appRecord.ReviewsStatus = "Error";
mongoClient.SaveRecord <AppModel> (appRecord);
}
}
}
}
static void Main (string[] args)
{
// Configuring Log Object
Logger logger = LogManager.GetCurrentClassLogger ();
// Control Variable (Bool - Should the process use proxies? )
bool isUsingProxies = false;
logger.Info ("Checking proxies configuration");
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
isUsingProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists (fPath))
{
logger.Fatal ("Couldnt find proxies on path : " + fPath);
System.Environment.Exit (-100);
}
// Reading Proxies from File
logger.Info ("Loading Proxies");
string[] fLines = File.ReadAllLines (fPath, Encoding.GetEncoding ("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load (fLines.ToList ());
}
catch (Exception ex)
{
logger.Fatal (ex);
System.Environment.Exit (-101);
}
}
// MongoDB instance Creation
logger.Info ("Configuring MonboDB Client");
// Creating instance of Mongo Handler for the main collection
MongoDBWrapper mongoClient = new MongoDBWrapper ();
string fullServerAddress = String.Join (":", Consts.MONGO_SERVER, Consts.MONGO_PORT);
mongoClient.ConfigureDatabase (Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, fullServerAddress, Consts.MONGO_TIMEOUT, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION);
logger.Info ("Iterating over Apps");
// Creating Play Store Parser
PlayStoreParser parser = new PlayStoreParser ();
// App Model
AppModel appRecord;
// Control Variable
bool noError = true;
// Finding all the "Apps" that didn't have the reviews visited yet
while ((appRecord = mongoClient.FindAndModifyReviews ()) != null)
{
// Extracting app ID from URL
string appId = appRecord.Url.Replace (Consts.PLAY_STORE_PREFIX, String.Empty);
// Console Feedback
logger.Info ("Processing App [ " + appRecord.Name + " ] ");
try
{
// Console Feedback
Console.Write ("Reviews from : " + appRecord.Name);
// Issuing Request for Reviews
string response = ReviewsWrapper.GetAppReviews (appId, 1, isUsingProxies);
// Checking for Blocking Situation
if (String.IsNullOrEmpty(response))
{
logger.Info ("Blocked by Play Store. Sleeping process for 10 minutes before retrying.");
// Thread Wait for 10 seconds
Thread.Sleep (TimeSpan.FromSeconds (10));
}
// Checking for "No Reviews" app
if (response.Length < 50)
{
logger.Info ("No Reviews for this app. Skipping");
Console.Write (" - No Reviews Found\n");
continue;
}
// Normalizing Response to Proper HTML
response = ReviewsWrapper.NormalizeResponse (response);
// List of Reviews
List<AppReview> reviews = new List<AppReview> ();
// Iterating over Parsed Reviews
foreach (var review in parser.ParseReviews (response))
{
// Adding App Data to the review
review.appID = appId;
review.appName = appRecord.Name;
review.appURL = appRecord.Url;
// Capture Timestamp to the model
review.timestamp = DateTime.Now;
// Adding reviews to the current list
reviews.Add (review);
}
// Any Review Found ?
if (reviews.Count > 0)
{
Console.Write (" - " + reviews.Count + " Reviews Found\n");
// Checking if there was any previous list of reviews
if (appRecord.Reviews == null)
{
appRecord.Reviews = reviews;
}
else // Previous List found - Appending only the new ones
{
foreach (var review in reviews)
{
if (!appRecord.Reviews.Any (t => t.permalink.Equals (review.permalink)))
{
appRecord.Reviews.Add (review);
}
}
}
}
}
catch (Exception ex)
{
logger.Error (ex);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine ("Error : " + ex.Message);
Console.ForegroundColor = ConsoleColor.White;
noError = false;
}
finally
{
// Toggling back the "ReviewsStatus" attribute from the model
if (noError)
{
appRecord.ReviewsStatus = "Visited";
mongoClient.SaveRecord<AppModel> (appRecord);
}
else // "Error" status
{
appRecord.ReviewsStatus = "Error";
mongoClient.SaveRecord<AppModel> (appRecord);
}
}
}
}