public void PurgeQueueTest()
{
QueueMessageBatchTest();
AWSSQSHelper queueTestAWSSQS = new AWSSQSHelper(queueName, 10, regionEndPoint, myAccessKey, mySecretKey);
queueTestAWSSQS.PurgeQueue();
}
static void CreateSQSQueueTest()
{
int AmazonSQSMaxMessageSize = 256 * 1024;
String ErrorMessage = "Fail";
int DelaySeconds = 1;
int MaximumMessageSize = AmazonSQSMaxMessageSize;
int MessageRetentionPeriod = 60 * 60 * 24 * 14; // 14 days
int ReceiveMessageWaitTimeSeconds = 1;
int VisibilityTimeout = 43200;
string Policy = "";
Assert.True(AWSSQSHelper.CreateSQSQueue(queueName, regionEndPoint, out ErrorMessage, DelaySeconds, MaximumMessageSize, MessageRetentionPeriod,
ReceiveMessageWaitTimeSeconds, VisibilityTimeout, Policy, myAccessKey, mySecretKey));
}
static void TestCreateSQSQueue()
{
String name = "queueTestAWSSQS";
var regionEndPoint = AWSGeneralHelper.GetRegionEndpoint();
String ErrorMessage = "Fail";
int DelaySeconds = 900;
int MaximumMessageSize = AmazonSQSMaxMessageSize;
int MessageRetentionPeriod = 60;
int ReceiveMessageWaitTimeSeconds = 20;
int VisibilityTimeout = 43200;
string Policy = "";
bool hasSucceed = false;
hasSucceed = AWSSQSHelper.CreateSQSQueue(name, regionEndPoint, out ErrorMessage, DelaySeconds, MaximumMessageSize, MessageRetentionPeriod,
ReceiveMessageWaitTimeSeconds, VisibilityTimeout, Policy);
}
public void QueueMessageBatchTest()
{
// Creating my messages list
IList <string> messages = new List <string> ();
for (int ind = 1; ind <= 10; ind++)
{
string body = "This is message number " + ind + " ! ";
messages.Add(body);
}
// Accessing the queue
AWSSQSHelper queueTestAWSSQS = new AWSSQSHelper(queueName, 10, regionEndPoint, myAccessKey, mySecretKey);
// Adding messages
Assert.True(queueTestAWSSQS.EnqueueMessages(messages));
}
public void GetQueueMessagesBatchTest()
{
AWSSQSHelper queueTestAWSSQS = new AWSSQSHelper(queueName, 10, regionEndPoint, myAccessKey, mySecretKey);
// Requesting messages
queueTestAWSSQS.DeQueueMessages();
// Asserting they are the right ones
int tamp = 1;
foreach (var msg in queueTestAWSSQS.GetMessages())
{
string msgbody = "This is message number " + tamp + " ! ";
Assert.True(msg.Equals(msgbody));
tamp += 1;
//Deleting messages
queueTestAWSSQS.DeleteMessage(msg);
}
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Reading Configuration");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret);
// Step 1 - Trying to obtain the root page html (source of all the apps)
var rootPageResponse = httpClient.GetRootPage();
// Sanity Check
if (String.IsNullOrWhiteSpace(rootPageResponse))
{
_logger.LogMessage("Error obtaining Root Page HTMl - Aborting", "Timeout Error");
return;
}
// Step 2 - Extracting Category Urls from the Root Page and queueing their Urls
foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse))
{
// Logging Feedback
_logger.LogMessage("Queueing Category : " + categoryUrl);
// Queueing Category Urls
sqsWrapper.EnqueueMessage(categoryUrl);
}
_logger.LogMessage("End of Bootstrapping phase");
}
public void QueueMessageTest()
{
String msgbody = "First test To Queue";
// Accessing the Queue
AWSSQSHelper queueTestAWSSQS = new AWSSQSHelper(queueName, 2, regionEndPoint, myAccessKey, mySecretKey);
// Adding msgs
Assert.True(queueTestAWSSQS.EnqueueMessage(msgbody));
// Request messages
queueTestAWSSQS.DeQueueMessages();
// Have i receved any ?
Assert.True(queueTestAWSSQS.AnyMessageReceived());
// Asserting they are the right one
foreach (var msg in queueTestAWSSQS.GetMessages())
{
Assert.True(msg.Equals(msgbody));
queueTestAWSSQS.DeleteMessage(msg);
}
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
// Loading Configuration
LogSetup.InitializeLog("Apple_Store_Urls_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Loading Config
_logger.Info("Loading Configurations from App.config");
LoadConfiguration();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
_logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
_logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// AWS Queue Handler
_logger.Info("Initializing Queues");
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info("Started Processing Individual Apps Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
{
bool processingWorked = true;
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
//appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8";
htmlResponse = httpClient.Get(appUrl.Body, shouldUseProxies);
if (String.IsNullOrEmpty(htmlResponse))
{
// Extending Fallback time
retries++;
int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;
_logger.Info("Retrying Request for App Page [ " + sleepTime / 1000 + " ]");
Thread.Sleep(sleepTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
continue;
}
// Feedback
_logger.Info("Current page " + appUrl.Body, "Parsing App Data");
// Parsing Data out of the Html Page
AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
parsedApp.url = appUrl.Body;
// Enqueueing App Data
appsDataQueue.EnqueueMessage(parsedApp.ToJson());
// Little Hiccup
Thread.Sleep(_hiccupTime);
}
catch (Exception ex)
{
_logger.Error(ex);
// Setting Flag to "False"
processingWorked = false;
}
finally
{
//Deleting the message - Only if the processing worked
if (processingWorked)
{
appsUrlQueue.DeleteMessage(appUrl);
}
}
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
} while (true);
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
// Setting Up Log
LogSetup.InitializeLog("Apple_Store_Crawler.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Starting Flow
_logger.Info("Worker Started");
// Loading Configuration
_logger.Info("Reading Configuration");
LoadConfiguration();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
_logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
_logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// AWS Queue Handler
_logger.Info("Initializing Queues");
AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret);
// Step 1 - Trying to obtain the root page html (source of all the apps)
var rootPageResponse = httpClient.GetRootPage(shouldUseProxies);
// Sanity Check
if (String.IsNullOrWhiteSpace(rootPageResponse))
{
_logger.Info("Error obtaining Root Page HTMl - Aborting", "Timeout Error");
return;
}
// Step 2 - Extracting Category Urls from the Root Page and queueing their Urls
foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse))
{
// Logging Feedback
_logger.Info("Queueing Category : " + categoryUrl);
// Queueing Category Urls
sqsWrapper.EnqueueMessage(categoryUrl);
}
_logger.Info("End of Bootstrapping phase");
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing Individual Apps Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
{
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
//appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8";
htmlResponse = httpClient.Get(appUrl.Body);
if (String.IsNullOrEmpty(htmlResponse))
{
// Extending Fallback time
retries++;
int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;
_logger.LogMessage("Retrying Request for App Page [ " + sleepTime / 1000 + " ]", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
Thread.Sleep(sleepTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
//appsUrlQueue.DeleteMessage (appUrl);
continue;
}
// Feedback
_logger.LogMessage("Current page " + appUrl.Body, "Parsing App Data");
// Parsing Data out of the Html Page
AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
parsedApp.url = appUrl.Body;
// Enqueueing App Data
appsDataQueue.EnqueueMessage(parsedApp.ToJson());
// Little Hiccup
Thread.Sleep(_hiccupTime);
}
catch (Exception ex)
{
_logger.LogMessage(ex.Message, "App Url Processing", BDC.BDCCommons.TLogEventLevel.Error);
}
finally
{
//Deleting the message
appsUrlQueue.DeleteMessage(appUrl);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}
static void Main(string[] args)
{
// Loading Configuration
LogSetup.InitializeLog("Apple_Store_Recorder.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Loading Config
_logger.Info("Loading Configurations from App.config");
LoadConfiguration();
// Initializing Queue
_logger.Info("Initializing Queue");
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper backup = new AWSSQSHelper("DeadLetter", _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Creating MongoDB Instance
_logger.Info("Loading MongoDB / Creating Instances");
MongoDBWrapper mongoDB = new MongoDBWrapper();
string serverAddr = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT);
mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, serverAddr, 10000, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
// Buffer of Messages to be recorder
List <AppleStoreAppModel> recordsBuffer = new List <AppleStoreAppModel> ();
List <Message> messagesBuffer = new List <Message> ();
// Insert Batch Size
int batchSize = 1000;
_logger.Info("Started Recording App Data");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsDataQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsDataQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appDataMessage in appsDataQueue.GetDequeuedMessages())
{
try
{
// Deserializing message
var appData = AppleStoreAppModel.FromJson(appDataMessage.Body);
// Dumping "Url" to "_id"
appData._id = appData.url;
// Adding it to the buffer of records to be recorded
recordsBuffer.Add(appData);
// Adding message to the buffer of messages to be deleted
messagesBuffer.Add(appDataMessage);
// Is it time to batch insert ?
if ((recordsBuffer.Count % batchSize) == 0)
{
// Batch Insertion
mongoDB.BatchInsert <AppleStoreAppModel> (recordsBuffer);
// Logging Feedback
_logger.Info("\tApps Recorded : " + recordsBuffer.Count);
// Deleting Messages
messagesBuffer.ForEach((msg) => appsDataQueue.DeleteMessage(msg));
_logger.Info("\tMessages Deleted: " + messagesBuffer.Count);
// Clearing Buffers
recordsBuffer.Clear();
messagesBuffer.Clear();
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
finally
{
// Deleting the message
appsDataQueue.DeleteMessage(appDataMessage);
backup.EnqueueMessage(appDataMessage.Body);
}
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
} while (true);
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper charactersUrlQueue = new AWSSQSHelper(_characterUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper numericUrlQueue = new AWSSQSHelper(_numericUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing Character Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!charactersUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!charactersUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var characterUrl in charactersUrlQueue.GetDequeuedMessages())
{
// Console Feedback
_logger.LogMessage("Started Parsing Url : " + characterUrl.Body);
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get(characterUrl.Body);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.LogMessage("Retrying Request for Character Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
retries++;
// Small Hiccup
Thread.Sleep(_hiccupTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
charactersUrlQueue.DeleteMessage(characterUrl);
continue;
}
// Hashset of urls processed (to avoid duplicates)
HashSet <String> urlsQueued = new HashSet <String> ();
// Executing Request and Queueing Urls until there's no other Url to be queued
do
{
// Flag to check whether any url was added after the last iteration (avoids endless loop)
bool anyNewUrl = false;
// If the request worked, parses the Urls out of the page
foreach (string numericUrls in parser.ParseNumericUrls(htmlResponse).Select(t => HttpUtility.HtmlDecode(t)))
{
// Checking if this url was previously queued
if (!urlsQueued.Contains(numericUrls))
{
// Enqueueing Urls
numericUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(numericUrls));
// Adding url to the local hashset
urlsQueued.Add(numericUrls);
anyNewUrl = true;
}
}
// Checking for the need to perform another http request for the next page
if (parser.IsLastPage(htmlResponse) || !anyNewUrl)
{
break; // Breaks "While" Loop
}
// Feedback
_logger.LogMessage("Urls Queued For This Page : " + urlsQueued.Count, "\n\tProcessing Feedback");
// If it got to this point, it means that there are more pages to be processed
// Parsing URL of the "Last" page (the last that's visible)
string lastPageUrl = HttpUtility.HtmlDecode(parser.ParseLastPageUrl(htmlResponse));
// Executing Http Request for this Url (with retries)
retries = 0;
do
{
// HTTP Get for the Page
htmlResponse = httpClient.Get(lastPageUrl);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.LogMessage("Retrying Request for Last Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
retries++;
// Small Hiccup
Thread.Sleep(_hiccupTime);
}
} while (String.IsNullOrEmpty(htmlResponse) && retries <= _maxRetries);
} while (true);
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
finally
{
charactersUrlQueue.DeleteMessage(characterUrl);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}
public Worker(IAmazonSQS amazonSQS, AWSSQSHelper helper, IMediator mediator)
{
_amazonSQS = amazonSQS ?? throw new ArgumentNullException(nameof(amazonSQS));
_helper = helper ?? throw new ArgumentNullException(nameof(helper));
_mediator = mediator ?? throw new ArgumentNullException(nameof(mediator));
}
static void Main(string[] args)
{
// Creating needed Instances
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// Initializing Queue
_logger.LogMessage("Initializing Queue");
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Creating MongoDB Instance
_logger.LogMessage("Loading MongoDB / Creating Instances");
MongoDBWrapper mongoDB = new MongoDBWrapper();
string serverAddr = String.Join(":", Consts.MONGO_SERVER, Consts.MONGO_PORT);
mongoDB.ConfigureDatabase(Consts.MONGO_USER, Consts.MONGO_PASS, Consts.MONGO_AUTH_DB, serverAddr, 10000, Consts.MONGO_DATABASE, Consts.MONGO_COLLECTION);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing App Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsDataQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsDataQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appDataMessage in appsDataQueue.GetDequeuedMessages())
{
try
{
// Deserializing message
var appData = AppleStoreAppModel.FromJson(appDataMessage.Body);
// Checking for duplicates
if (!mongoDB.IsAppOnDatabase <AppleStoreAppModel> (appData.url))
{
// Recording App Data
mongoDB.Insert <AppleStoreAppModel> (appData);
}
}
catch (Exception ex)
{
_logger.LogMessage(ex.Message, "App Recording", BDC.BDCCommons.TLogEventLevel.Error);
}
finally
{
// Deleting the message
appsDataQueue.DeleteMessage(appDataMessage);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}
static void DeleteSQSQueueTest()
{
Assert.False(AWSSQSHelper.DestroySQSQueue(queueName, regionEndPoint, out ErrorMessage, myAccessKey, mySecretKey));
}
//public CreateCustomerCommandHandler(IAmazonSimpleNotificationService amazonSimpleNotificationService, ICustomerRepository customerRepository)
//{
// _amazonSimpleNotificationService = amazonSimpleNotificationService ?? throw new ArgumentNullException(nameof(amazonSimpleNotificationService));
// _customerRepository = customerRepository ?? throw new ArgumentNullException(nameof(customerRepository));
//}
public CreateCustomerCommandHandler(IAmazonSQS amazonSQS, ICustomerRepository customerRepository, AWSSQSHelper helper)
{
_amazonSQS = amazonSQS ?? throw new ArgumentNullException(nameof(amazonSQS));
_customerRepository = customerRepository ?? throw new ArgumentNullException(nameof(customerRepository));
_helper = helper ?? throw new ArgumentNullException(nameof(helper));
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
// Loading Configuration
LogSetup.InitializeLog("Apple_Store_Numerics_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Loading Config
_logger.Info("Loading Configurations from App.config");
LoadConfiguration();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
_logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
_logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// AWS Queue Handler
_logger.Info("Initializing Queues");
AWSSQSHelper numericUrlQueue = new AWSSQSHelper(_numericUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info("Started Processing Numeric Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!numericUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!numericUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var numericUrl in numericUrlQueue.GetDequeuedMessages())
{
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get(numericUrl.Body, shouldUseProxies);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.Info("Retrying Request for Category Page");
retries++;
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
numericUrlQueue.DeleteMessage(numericUrl);
continue;
}
// Feedback
_logger.Info("Current page " + numericUrl.Body);
foreach (var parsedAppUrl in parser.ParseAppsUrls(htmlResponse))
{
// Enqueueing App Urls
appsUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(parsedAppUrl));
}
}
catch (Exception ex)
{
_logger.Info(ex);
}
finally
{
// Deleting the message
numericUrlQueue.DeleteMessage(numericUrl);
}
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
} while (true);
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper categoriesUrlQueue = new AWSSQSHelper(_categoriesQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper charactersUrlQueue = new AWSSQSHelper(_characterUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing Category Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!categoriesUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!categoriesUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var categoryUrl in categoriesUrlQueue.GetDequeuedMessages())
{
// Console Feedback
_logger.LogMessage("Started Parsing Category : " + categoryUrl.Body);
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get(categoryUrl.Body);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.LogMessage("Retrying Request for Category Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
retries++;
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
categoriesUrlQueue.DeleteMessage(categoryUrl);
continue;
}
// If the request worked, parses the urls out of the page
foreach (string characterUrls in parser.ParseCharacterUrls(htmlResponse))
{
// Enqueueing Urls
charactersUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(characterUrls));
}
}
catch (Exception ex)
{
_logger.LogMessage(ex.Message, "Category Url Processing", BDC.BDCCommons.TLogEventLevel.Error);
}
finally
{
// Deleting the message
categoriesUrlQueue.DeleteMessage(categoryUrl);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}
