static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
// Loading Configuration
LogSetup.InitializeLog("Apple_Store_Urls_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Loading Config
_logger.Info("Loading Configurations from App.config");
LoadConfiguration();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
_logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
_logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// AWS Queue Handler
_logger.Info("Initializing Queues");
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info("Started Processing Individual Apps Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
{
if (appUrl.Body.IndexOf("https://itunes.apple.com/us/app") < 0 && appUrl.Body.IndexOf("https://itunes.apple.com/app/") < 0)
{
_logger.Info("Invalid Message. Deleting [{0}]", appUrl.Body);
appsUrlQueue.DeleteMessage(appUrl);
continue;
}
bool processingWorked = true;
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get(appUrl.Body, shouldUseProxies);
if (String.IsNullOrEmpty(htmlResponse))
{
// Extending Fallback time
retries++;
int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;
_logger.Info("Retrying Request for App Page [ " + sleepTime / 1000 + " ]");
Thread.Sleep(sleepTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
continue;
}
// Feedback
_logger.Info("Current page " + appUrl.Body, "Parsing App Data");
// Parsing Data out of the Html Page
AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
parsedApp.url = appUrl.Body;
// Enqueueing App Data
appsDataQueue.EnqueueMessage(parsedApp.ToJson());
// Little Hiccup
Thread.Sleep(_hiccupTime);
}
catch (Exception ex)
{
_logger.Error(ex);
// Setting Flag to "False"
processingWorked = false;
}
finally
{
//Deleting the message - Only if the processing worked
if (processingWorked)
{
appsUrlQueue.DeleteMessage(appUrl);
}
}
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
} while (true);
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing Individual Apps Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
{
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
//appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8";
htmlResponse = httpClient.Get(appUrl.Body);
if (String.IsNullOrEmpty(htmlResponse))
{
// Extending Fallback time
retries++;
int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;
_logger.LogMessage("Retrying Request for App Page [ " + sleepTime / 1000 + " ]", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
Thread.Sleep(sleepTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
//appsUrlQueue.DeleteMessage (appUrl);
continue;
}
// Feedback
_logger.LogMessage("Current page " + appUrl.Body, "Parsing App Data");
// Parsing Data out of the Html Page
AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
parsedApp.url = appUrl.Body;
// Enqueueing App Data
appsDataQueue.EnqueueMessage(parsedApp.ToJson());
// Little Hiccup
Thread.Sleep(_hiccupTime);
}
catch (Exception ex)
{
_logger.LogMessage(ex.Message, "App Url Processing", BDC.BDCCommons.TLogEventLevel.Error);
}
finally
{
//Deleting the message
appsUrlQueue.DeleteMessage(appUrl);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}