public async Task <ActionResult <Response <bool> > > ClearAssignedRequestsFromPreparer(int preparer)
{
logger.LogInformation("POST cabe/asignados/{preparador}/clear invocado con preparador " + preparer);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var assignedRequests = await potigianContext.RequestHeaders
.Where(req =>
(req.PreparerCode == preparer.ToString() || req.PreparerCode == $"CSPRP{preparer}") && req.SituationCode == Config.Requests.StateInPreparation)
.ToListAsync();
assignedRequests.ForEach(r =>
{
r.SituationCode = Config.Requests.StateAvailableToPrepare;
r.SituationDate = DateTime.Now;
r.PreparerCode = null;
potigianContext.Update(r);
});
await potigianContext.SaveChangesAsync();
return true;
}));
}
public async Task <ActionResult <Response <List <Preparer> > > > GetPreparers()
{
logger.LogInformation("GET preparadores invocado");
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() => await potigianContext.Preparers.ToListAsync()));
}
private static void TestAlbumsServicesWithJson(string baseUri, HttpClient client)
{
IRequestsHandler albumsHandler = new RequestsHandler(baseUri, "albums", client, "application/json");
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get all albums...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(albumsHandler.Get());
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get album by id...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(albumsHandler.GetById(1));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to add album...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(albumsHandler.Post("{\"title\" : \"Asdf\", \"producerId\" : 1 }"));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to edit album...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(albumsHandler.Put(1, "{\"title\" : \"Asdf\", \"producerId\" : 1 }"));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to delete album...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
albumsHandler.Delete(5);
}
public async Task <ActionResult <Response <List <RequestDetails> > > > GetRequestDetails(int prefixDoc, int doc, int suffixDoc)
{
logger.LogInformation("GET detalle/{prefijo}/{doc}/{sufijo} invocado con prefijo " + prefixDoc + ", doc " + doc + " y sufijo " + suffixDoc);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var requestDetails = await potigianContext.RequestDetails
.Where(req =>
req.DocumentPrefix == prefixDoc &&
req.DocumentCode == doc &&
req.DocumentSuffix == suffixDoc)
.Join(
potigianContext.Articles,
req => req.ArticleCode,
article => article.Code,
(req, article) => req.Append(article))
.ToListAsync();
foreach (var req in requestDetails)
{
var value = await potigianContext.BranchArticles.FirstOrDefaultAsync(a => a.ArticleCode == req.ArticleCode);
if (value.ArticleCode == null || value.ArticleCode == 0)
{
value.ArticleCode = 1;
}
req.Append(value);
}
return requestDetails;
}));
}
private static void TestSongsServicesWithJson(string baseUri, HttpClient client)
{
IRequestsHandler songsHandler = new RequestsHandler(baseUri, "songs", client, "application/json");
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get all songs...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(songsHandler.Get());
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get song by id...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(songsHandler.GetById(1));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to add song...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(songsHandler.Post("{ \"title\": \"Postman\", \"artistId\" : 1, \"albumId\": 1 }"));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to edit song...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(songsHandler.Put(1, "{ \"title\": \"Postman\", \"artistId\" : 1, \"albumId\": 1 }"));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to delete song...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
songsHandler.Delete(5);
}
public async Task <ActionResult <Response <List <RequestDetails> > > > GetRequestsDetails()
{
logger.LogInformation("GET detalle invocado");
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() => await potigianContext.RequestDetails.ToListAsync()));
}
public async Task <ActionResult <Response <List <PurchaseOrderHeader> > > > GetOrdersByProvider(int code)
{
logger.LogInformation("GET proveedor/{code} invocado con code " + code);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() => await potigianContext.PurchaseOrdersHeaders.Where(o => o.ProviderCode == code && o.Situation == 1).ToListAsync()));
}
private static void TestArtistsServicesWithJson(string baseUri, HttpClient client)
{
IRequestsHandler artistsHandler = new RequestsHandler(baseUri, "artists", client, "application/json");
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get all artists...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(artistsHandler.Get());
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get artist by id...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(artistsHandler.GetById(1));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to add artist...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(artistsHandler.Post("{ \"name\" : \"Gosho\" }"));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to edit artist...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(artistsHandler.Put(1, "{ \"name\" : \"Gosho\" }"));
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to delete artist...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
artistsHandler.Delete(5);
}
public async Task <ActionResult <Response <List <RequestHeaders> > > > GetRequestsHeadersByPreparer(int preparer)
{
logger.LogInformation("GET cabe/asignados/{preparador} invocado con preparador " + preparer);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() => await potigianContext.RequestHeaders
.Where(h => (h.PreparerCode == preparer.ToString() || h.PreparerCode == $"CSPRP{preparer}") && h.SituationCode == Config.Requests.StateInPreparation)
.ToListAsync()));
}
public async Task <ActionResult <Response <List <RequestHeaders> > > > GetRequestsHeadersBySituation(int situation)
{
logger.LogInformation("GET cabe/situacion/{code} invocado con code " + situation);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() => await potigianContext.RequestHeaders
.Where(h => h.SituationCode == situation)
.ToListAsync()));
}
public async Task <ActionResult <Response <bool> > > StartRequest(int prefixDoc, int doc, int suffixDoc, int preparer)
{
logger.LogInformation("POST cabe/asignados/{prefijo}/{doc}/{sufijo}/start/{preparador} invocado con prefijo " + prefixDoc + ", doc " + doc + ", sufijo " + suffixDoc + " y preparador " + preparer);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var requestPreparations = potigianContext.RequestPreparations.Where(rp => rp.DocumentSuffix == suffixDoc);
if (await requestPreparations.CountAsync() == 0) // new entry
{
var preparation = new RequestPreparation
{
Code = preparer.ToString(),
DocumentSuffix = suffixDoc,
InsertDate = DateTime.Now,
StartDate = DateTime.Now,
StatusCode = Config.Requests.StateInPreparation,
EndDate = null,
};
potigianContext.Add(preparation);
potigianContext.Add(PreparerProductivity.FromPreparation(preparation));
await potigianContext.SaveChangesAsync();
}
else
{
try
{
var preparation = await requestPreparations.FirstAsync();
if (preparation.StartDate == null)
{
preparation.Code = preparer.ToString();
preparation.StartDate = DateTime.Now;
preparation.EndDate = null;
preparation.StatusCode = Config.Requests.StateInPreparation;
potigianContext.Update(preparation);
potigianContext.Add(PreparerProductivity.FromPreparation(preparation));
await potigianContext.SaveChangesAsync();
}
}
catch (Exception ex)
{
throw ex;
}
}
return true;
}));
}
private static void TestAlbumsServicesWithXml(string baseUri, HttpClient client)
{
IRequestsHandler albumsHandler = new RequestsHandler(baseUri, "albums", client, "application/xml");
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get all albums...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(albumsHandler.Get());
Console.WriteLine("\n=================================\n");
Console.WriteLine("Press Enter to get album by id...");
Console.WriteLine("\n=================================\n");
Console.ReadKey();
Console.WriteLine(albumsHandler.GetById(1));
}
public ConnectionManager(
ITransportResource transport,
CommandBuffer commandBuffer,
CommandParser commandParser,
RequestsHandler requestsHandler,
IDtoSerializer serializer)
{
_transport = transport;
_commandBuffer = commandBuffer;
_commandParser = commandParser;
_requestsHandler = requestsHandler;
_serializer = serializer;
_commandBuffer.CommandAssembled += CommandBuffer_CommandAssembled;
_transport.DataReceived += data => _commandBuffer.AppendBytes(data);
_transport.ConnectionError += Transport_ConnectionError;
}
public async Task <ActionResult <Response <Preparer> > > GetPreparer(int id)
{
logger.LogInformation("GET preparadores/{id} invocado con id " + id);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var preparer = await potigianContext.Preparers.FirstOrDefaultAsync(p => p.Id == id);
if (preparer == null)
{
preparer = new Preparer
{
Code = "-1"
};
}
return preparer;
}));
}
public async Task <ActionResult <Response <List <PurchaseOrderDetails> > > > GetOrderDetails(string prefixCode, string ocCode, string suffixCode)
{
logger.LogInformation("GET {prefijo}/{oc}/{sufijo} invocado con prefijo " + prefixCode + ", oc " + ocCode + " y sufijo " + suffixCode);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() => await potigianContext.PurchaseOrderDetails
.Where(od =>
od.PrefixOcCode.ToString() == prefixCode &&
od.OcCode.ToString() == ocCode &&
od.SuffixOcCode.ToString() == suffixCode)
.Join(
potigianContext.Articles,
od => od.ArticleCode,
article => article.Code,
(od, article) => od.Append(article))
.Join(
potigianContext.ProviderArticles,
od => od.ArticleCode,
providerArticle => providerArticle.ArticleCode,
(od, providerArticle) => od.Append(providerArticle))
.ToListAsync()));
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Reading Configuration");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret);
// Step 1 - Trying to obtain the root page html (source of all the apps)
var rootPageResponse = httpClient.GetRootPage();
// Sanity Check
if (String.IsNullOrWhiteSpace(rootPageResponse))
{
_logger.LogMessage("Error obtaining Root Page HTMl - Aborting", "Timeout Error");
return;
}
// Step 2 - Extracting Category Urls from the Root Page and queueing their Urls
foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse))
{
// Logging Feedback
_logger.LogMessage("Queueing Category : " + categoryUrl);
// Queueing Category Urls
sqsWrapper.EnqueueMessage(categoryUrl);
}
_logger.LogMessage("End of Bootstrapping phase");
}
public async Task <ActionResult <Response <User> > > LoginUser(string usr, string code)
{
logger.LogInformation($"GET de usuario invocado por " + usr);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var user = await(from u in securityContext.Users
join s in securityContext.AccessSystems on u.AccessCode.Trim() equals s.AccessCode.Trim()
where s.Name.Trim() == "Mercaderias" && u.Code.Trim() == usr && u.AccessCode.Trim() == code
select u).FirstOrDefaultAsync();
if (user == null)
{
user = new User
{
Code = "-1"
};
}
return user;
}));
}
public async Task <ActionResult <Response <List <PurchaseOrderHeader> > > > GetOrdersByProviderAndSuffixCode(string providerCode, string suffixCode)
{
logger.LogInformation("GET {sufijo}/proveedor/{code} invocado con sufijo " + suffixCode + " y code " + providerCode);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var expr = default(Expression <Func <PurchaseOrderHeader, bool> >);
if (providerCode == "none" && suffixCode == "none")
{
expr = o => o.Situation == 1;
}
else if (providerCode == "none")
{
expr = o => o.OrderSuffix.ToString().Contains(suffixCode) && o.Situation == 1;
}
else if (suffixCode == "none")
{
expr = o => o.ProviderCode.ToString().Contains(providerCode) && o.Situation == 1;
}
else
{
expr = o => o.OrderSuffix.ToString().Contains(suffixCode) && o.ProviderCode.ToString().Contains(providerCode) && o.Situation == 1;
}
var ocs = potigianContext.PurchaseOrdersHeaders.Where(expr);
await ocs.ForEachAsync(async oc =>
{
oc.Items = await potigianContext.PurchaseOrderDetails
.Where(od => od.SuffixOcCode == oc.OrderSuffix && od.PrefixOcCode == oc.OrderPrefix && od.OcCode == oc.OrderCode)
.CountAsync();
});
return await ocs.ToListAsync();
}));
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
// Loading Configuration
LogSetup.InitializeLog("Apple_Store_Numerics_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Loading Config
_logger.Info("Loading Configurations from App.config");
LoadConfiguration();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
_logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
_logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// AWS Queue Handler
_logger.Info("Initializing Queues");
AWSSQSHelper numericUrlQueue = new AWSSQSHelper(_numericUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info("Started Processing Numeric Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!numericUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!numericUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var numericUrl in numericUrlQueue.GetDequeuedMessages())
{
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get(numericUrl.Body, shouldUseProxies);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.Info("Retrying Request for Category Page");
retries++;
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
numericUrlQueue.DeleteMessage(numericUrl);
continue;
}
// Feedback
_logger.Info("Current page " + numericUrl.Body);
foreach (var parsedAppUrl in parser.ParseAppsUrls(htmlResponse))
{
// Enqueueing App Urls
appsUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(parsedAppUrl));
}
}
catch (Exception ex)
{
_logger.Info(ex);
}
finally
{
// Deleting the message
numericUrlQueue.DeleteMessage(numericUrl);
}
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
} while (true);
}
public async Task <ActionResult <Response <List <RequestHeaders> > > > AssignRequestsHeadersToPreparer(
int preparer,
[FromQuery(Name = "cigarrillos")] bool cigarettesOnly,
[FromQuery(Name = "pedido")] int suffixNumber,
[FromQuery(Name = "reparto")] int deliveryNumber)
{
logger.LogInformation("POST cabe/asignados/{preparador} invocado con preparador " + preparer);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var assignedRequests = await potigianContext.RequestHeaders
.Where(req =>
(req.PreparerCode == preparer.ToString() || req.PreparerCode == $"CSPRP{preparer}") && req.SituationCode == Config.Requests.StateInPreparation)
.ToListAsync();
// If there are any assigned requests, we won't assign new ones.
if (assignedRequests.Count > 0)
{
return assignedRequests;
}
var newRequests = default(List <RequestHeaders>);
if (suffixNumber != 0)
{
newRequests = await potigianContext.RequestHeaders
.Where(req => req.DocumentSuffix == suffixNumber && req.SituationCode == Config.Requests.StateAvailableToPrepare)
.ToListAsync();
}
else if (deliveryNumber != 0)
{
newRequests = await potigianContext.RequestHeaders
.Where(req => req.DistributionNumber == deliveryNumber && req.SituationCode == Config.Requests.StateAvailableToPrepare)
.ToListAsync();
}
if (newRequests == default(List <RequestHeaders>) || newRequests.Count == 0)
{
return new List <RequestHeaders>();
}
var suffixes = newRequests.Select(req => req.DocumentSuffix).Distinct();
var prefixes = newRequests.Select(req => req.DocumentPrefix).Distinct();
var docs = newRequests.Select(req => req.DocumentCode).Distinct();
var requestsDetails = await potigianContext.RequestDetails
.Where(req => suffixes.Contains(req.DocumentSuffix) &&
prefixes.Contains(req.DocumentPrefix) &&
docs.Contains(req.DocumentCode))
.ToListAsync();
var definedRequests = newRequests.Where(req =>
{
var details = requestsDetails
.Where(det => req.DocumentSuffix == det.DocumentSuffix &&
req.DocumentCode == det.DocumentCode &&
req.DocumentPrefix == det.DocumentPrefix);
bool areOnlyCigarettes = details.All(det => new List <decimal?>()
{
1, 2
}.Contains(det.FamilyCode));
return cigarettesOnly ? areOnlyCigarettes : !areOnlyCigarettes;
})
.Take(config.GetValue <int>("RequestsPerPreparer"))
.ToList();
definedRequests.ForEach(req =>
{
req.SituationCode = Config.Requests.StateInPreparation;
req.SituationDate = DateTime.Now;
req.PreparerCode = preparer.ToString();
potigianContext.Update(req);
});
await potigianContext.SaveChangesAsync();
return definedRequests;
}));
}
public async Task <ActionResult <Response <bool> > > CloseRequestPreparation(int prefixDoc, int doc, int suffixDoc, [FromBody] CloseRequestPayload payload)
{
logger.LogInformation("POST preparaciones/{prefijo}/{doc}/{sufijo} invocado con prefijo " + prefixDoc + ", doc " + doc + " y sufijo " + suffixDoc);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
bool closedComplete = true;
var requestDetails = await potigianContext.RequestDetails
.Where(req =>
req.DocumentPrefix == prefixDoc &&
req.DocumentCode == doc &&
req.DocumentSuffix == suffixDoc)
.ToListAsync();
var unfinishedRequestDetails = requestDetails
.Where(req => req.PackagesGrams != payload.ArticleCount[GetDictionaryCountKey(req)]);
var requestHeader = await potigianContext.RequestHeaders
.FirstAsync(req => req.DocumentPrefix == prefixDoc && req.DocumentCode == doc && req.DocumentSuffix == suffixDoc);
// Unfinished requests case
if (unfinishedRequestDetails.Count() > 0)
{
var missingRequestDetails = unfinishedRequestDetails
.Select(req => new RequestMissingDetails(req, payload.ArticleCount[GetDictionaryCountKey(req)]))
.ToList();
foreach (var request in unfinishedRequestDetails)
{
int count = payload.ArticleCount[GetDictionaryCountKey(request)];
if (count != 0)
{
request.PackagesGrams = count;
request.ArticleTotal = request.PackagesGrams *request.ArticleUnitaryPrice;
potigianContext.RequestDetails.Update(request);
}
}
potigianContext.RequestMissingDetails.AddRange(missingRequestDetails);
closedComplete = false;
}
var preparation = await potigianContext.RequestPreparations.FirstAsync(p => p.DocumentSuffix == suffixDoc);
preparation.EndDate = DateTime.Now;
preparation.StatusCode = Config.Requests.StateClosed;
preparation.MovementFlag = 'E';
var productivity = await potigianContext.PreparerProductivities.FirstAsync(p => p.DocumentSuffix == suffixDoc);
productivity.EndDate = preparation.EndDate;
productivity.StatusCode = preparation.StatusCode;
productivity.MovementFlag = preparation.MovementFlag;
var productivityDifferential = productivity.EndDate.Value.Subtract(productivity.StartDate.Value);
productivity.Days = productivityDifferential.Days;
productivity.Hours = productivityDifferential.Hours;
productivity.Minutes = productivityDifferential.Minutes;
productivity.Seconds = productivityDifferential.Seconds;
requestHeader.SituationCode = Config.Requests.StateClosed;
requestHeader.SituationDate = DateTime.Now;
requestHeader.Printer = payload.Printer;
requestHeader.TotalPackages = payload.Bags;
potigianContext.RequestHeaders.Update(requestHeader);
potigianContext.RequestPreparations.Update(preparation);
potigianContext.PreparerProductivities.Update(productivity);
await potigianContext.SaveChangesAsync();
return closedComplete;
}));
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
// Loading Configuration
LogSetup.InitializeLog("Apple_Store_Urls_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Loading Config
_logger.Info("Loading Configurations from App.config");
LoadConfiguration();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
_logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
_logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// AWS Queue Handler
_logger.Info("Initializing Queues");
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info("Started Processing Individual Apps Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
{
bool processingWorked = true;
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
//appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8";
htmlResponse = httpClient.Get(appUrl.Body, shouldUseProxies);
if (String.IsNullOrEmpty(htmlResponse))
{
// Extending Fallback time
retries++;
int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;
_logger.Info("Retrying Request for App Page [ " + sleepTime / 1000 + " ]");
Thread.Sleep(sleepTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
continue;
}
// Feedback
_logger.Info("Current page " + appUrl.Body, "Parsing App Data");
// Parsing Data out of the Html Page
AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
parsedApp.url = appUrl.Body;
// Enqueueing App Data
appsDataQueue.EnqueueMessage(parsedApp.ToJson());
// Little Hiccup
Thread.Sleep(_hiccupTime);
}
catch (Exception ex)
{
_logger.Error(ex);
// Setting Flag to "False"
processingWorked = false;
}
finally
{
//Deleting the message - Only if the processing worked
if (processingWorked)
{
appsUrlQueue.DeleteMessage(appUrl);
}
}
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
} while (true);
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper categoriesUrlQueue = new AWSSQSHelper(_categoriesQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper charactersUrlQueue = new AWSSQSHelper(_characterUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing Category Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!categoriesUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!categoriesUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var categoryUrl in categoriesUrlQueue.GetDequeuedMessages())
{
// Console Feedback
_logger.LogMessage("Started Parsing Category : " + categoryUrl.Body);
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get(categoryUrl.Body);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.LogMessage("Retrying Request for Category Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
retries++;
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
categoriesUrlQueue.DeleteMessage(categoryUrl);
continue;
}
// If the request worked, parses the urls out of the page
foreach (string characterUrls in parser.ParseCharacterUrls(htmlResponse))
{
// Enqueueing Urls
charactersUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(characterUrls));
}
}
catch (Exception ex)
{
_logger.LogMessage(ex.Message, "Category Url Processing", BDC.BDCCommons.TLogEventLevel.Error);
}
finally
{
// Deleting the message
categoriesUrlQueue.DeleteMessage(categoryUrl);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper charactersUrlQueue = new AWSSQSHelper(_characterUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper numericUrlQueue = new AWSSQSHelper(_numericUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing Character Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!charactersUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!charactersUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var characterUrl in charactersUrlQueue.GetDequeuedMessages())
{
// Console Feedback
_logger.LogMessage("Started Parsing Url : " + characterUrl.Body);
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get(characterUrl.Body);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.LogMessage("Retrying Request for Character Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
retries++;
// Small Hiccup
Thread.Sleep(_hiccupTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
charactersUrlQueue.DeleteMessage(characterUrl);
continue;
}
// Hashset of urls processed (to avoid duplicates)
HashSet <String> urlsQueued = new HashSet <String> ();
// Executing Request and Queueing Urls until there's no other Url to be queued
do
{
// Flag to check whether any url was added after the last iteration (avoids endless loop)
bool anyNewUrl = false;
// If the request worked, parses the Urls out of the page
foreach (string numericUrls in parser.ParseNumericUrls(htmlResponse).Select(t => HttpUtility.HtmlDecode(t)))
{
// Checking if this url was previously queued
if (!urlsQueued.Contains(numericUrls))
{
// Enqueueing Urls
numericUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(numericUrls));
// Adding url to the local hashset
urlsQueued.Add(numericUrls);
anyNewUrl = true;
}
}
// Checking for the need to perform another http request for the next page
if (parser.IsLastPage(htmlResponse) || !anyNewUrl)
{
break; // Breaks "While" Loop
}
// Feedback
_logger.LogMessage("Urls Queued For This Page : " + urlsQueued.Count, "\n\tProcessing Feedback");
// If it got to this point, it means that there are more pages to be processed
// Parsing URL of the "Last" page (the last that's visible)
string lastPageUrl = HttpUtility.HtmlDecode(parser.ParseLastPageUrl(htmlResponse));
// Executing Http Request for this Url (with retries)
retries = 0;
do
{
// HTTP Get for the Page
htmlResponse = httpClient.Get(lastPageUrl);
if (String.IsNullOrEmpty(htmlResponse))
{
_logger.LogMessage("Retrying Request for Last Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
retries++;
// Small Hiccup
Thread.Sleep(_hiccupTime);
}
} while (String.IsNullOrEmpty(htmlResponse) && retries <= _maxRetries);
} while (true);
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
finally
{
charactersUrlQueue.DeleteMessage(characterUrl);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}
public async Task <ActionResult <Response <SynchronizeOrderResponse> > > SynchronizeOrder(string prefixCode, string ocCode, string suffixCode, [FromBody] SynchronizeOrderPayload payload)
{
logger.LogInformation("POST {prefijo}/{oc}/{sufijo} invocado con prefijo " + prefixCode + ", oc " + ocCode + " y sufijo " + suffixCode);
return(await RequestsHandler.HandleAsyncRequest(
logger,
async() =>
{
var orderHeader = await potigianContext.PurchaseOrdersHeaders
.FirstAsync(h => h.OrderPrefix == decimal.Parse(prefixCode) && h.OrderSuffix == decimal.Parse(suffixCode) && h.OrderCode == decimal.Parse(ocCode));
var orderDetails = await potigianContext.PurchaseOrderDetails
.Where(d => d.PrefixOcCode == decimal.Parse(prefixCode) && d.SuffixOcCode == decimal.Parse(suffixCode) && d.OcCode == decimal.Parse(ocCode))
.ToListAsync();
var articleCodes = orderDetails.Select(d => d.ArticleCode);
var articles = await potigianContext.Articles.Where(a => articleCodes.Contains(a.Code)).ToListAsync();
var detailsPending = orderDetails.Where(d => d.RequestedPackages > payload.ArticleCount[d.ArticleCode]);
await potigianContext.Database.BeginTransactionAsync();
var response = new SynchronizeOrderResponse();
try
{
// CABE Procedure
{
var ocParam = new SqlParameter("@C_OC", value: decimal.Parse(ocCode));
var prefixParam = new SqlParameter("@U_PREFIJO_OC", value: decimal.Parse(prefixCode));
var suffixParam = new SqlParameter("@U_SUFIJO_OC", value: decimal.Parse(suffixCode));
var situationParam = new SqlParameter("@C_SITUAC_OC", value: (decimal)(detailsPending.Any() ? 1 : 2)); // 1 pending - 2 ready
var receivedGoodsDateParam = new SqlParameter("@f_comp_ing_merc", value: payload.OrderAdditionalInfo.Date);
var receivedGoodsCodeParam = new SqlParameter("@C_COMP_ING_MERC", value: (decimal)payload.OrderAdditionalInfo.BillType);
var purchasePrefixParam = new SqlParameter("@U_PREFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.PrefixOc);
var purchaseSuffixParam = new SqlParameter("@U_SUFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.SuffixOc);
var observationParam = new SqlParameter("@D_OBSERVACION_ING_MERC", value: payload.OrderAdditionalInfo.Observations);
var userParam = new SqlParameter("@vchUsuario", value: payload.OrderAdditionalInfo.User);
var terminalParam = new SqlParameter("@vchTerminal", value: payload.OrderAdditionalInfo.Terminal);
var outputMessageParam = new SqlParameter
{
ParameterName = "@vchMensaje",
Value = "",
Direction = ParameterDirection.Output,
SqlDbType = SqlDbType.VarChar,
Size = 255
};
var returnCode = new SqlParameter
{
ParameterName = "@returnCode",
Value = -1,
Direction = ParameterDirection.Output,
};
await potigianContext.Database.ExecuteSqlCommandAsync("EXEC @returnCode = dbo.SD03_OC_ING_MERC_ACTU_CABE " +
"@C_OC, @U_PREFIJO_OC, @U_SUFIJO_OC, @C_SITUAC_OC, @f_comp_ing_merc, @C_COMP_ING_MERC, " +
"@U_PREFIJO_COMP_ING_MERC, @U_SUFIJO_COMP_ING_MERC, @D_OBSERVACION_ING_MERC, " +
"@vchUsuario, @vchTerminal, @vchMensaje OUTPUT", returnCode, ocParam, prefixParam, suffixParam,
situationParam, receivedGoodsDateParam, receivedGoodsCodeParam, purchasePrefixParam,
purchaseSuffixParam, observationParam, userParam, terminalParam, outputMessageParam);
response.HeaderMessage = (string)outputMessageParam.Value;
response.HeaderReturnCode = (int)returnCode.Value;
}
// DETA PROCEDURE
foreach (var orderDetail in orderDetails)
{
var providerParam = new SqlParameter("@C_PROVEEDOR", value: orderHeader.ProviderCode);
var ocParam = new SqlParameter("@C_OC", value: decimal.Parse(ocCode));
var companyBranchParam = new SqlParameter("@C_SUCU_EMPR", value: payload.OrderAdditionalInfo.Branch);
var destinationBranchParam = new SqlParameter("@C_SUCU_DESTINO_ALT", value: orderHeader.AlternativeDestinationBranch);
var prefixOcParam = new SqlParameter("@U_PREFIJO_OC", value: decimal.Parse(prefixCode));
var suffixOcParam = new SqlParameter("@U_SUFIJO_OC", value: decimal.Parse(suffixCode));
var ocMotherParam = new SqlParameter("@M_OC_MADRE", value: orderHeader.MotherFlag);
var situationCodeParam = new SqlParameter("@C_SITUAC_OC", value: (decimal)(detailsPending.Any(d => d.ArticleCode == orderDetail.ArticleCode) ? 1 : 2)); // 1 pending - 2 ready
var articleCodeParam = new SqlParameter("@C_ARTICULO", value: orderDetail.ArticleCode);
var saleByWeightParam = new SqlParameter("@M_VENDE_POR_PESO", value: articles.First(a => a.Code == orderDetail.ArticleCode).SellByWeight);
var requestedUnitsParam = new SqlParameter("@Q_UNID_KGS_PED", value: orderDetail.RequestedPackages);
var deliveredUnitsParam = new SqlParameter("@Q_UNID_KGS_CUMPL", value: payload.ArticleCount[orderDetail.ArticleCode]);
var pendingUnitsParam = new SqlParameter("@Q_UNID_KGS_PEND", value: orderDetail.RequestedPackages - payload.ArticleCount[orderDetail.ArticleCode]);
var receivedUnitsParam = new SqlParameter("@Q_UNID_KGS_ING", value: orderDetail.RequestedPackages);
var receivedPacksParam = new SqlParameter("@Q_BULTOS_KGS_ING", value: orderDetail.RequestedBulks);
var receivedPiecesFactorParam = new SqlParameter("@Q_FACTOR_PIEZAS_ING", value: orderDetail.RequestedFactor);
var receivedGoodsBoughtParam = new SqlParameter("@C_COMP_ING_MERC", value: payload.OrderAdditionalInfo.BillType);
var receivedGoodsPrefixParam = new SqlParameter("@U_PREFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.PrefixOc);
var receivedGoodsSuffixParam = new SqlParameter("@U_SUFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.SuffixOc);
var goodsDateParam = new SqlParameter("@F_COMP_ING_MERC", value: payload.OrderAdditionalInfo.Date);
var programCodeParam = new SqlParameter("@C_PROGRAMA", value: Config.ProgramCode);
var observationParam = new SqlParameter("@D_OBSERVACION", value: payload.OrderAdditionalInfo.Observations);
var userParam = new SqlParameter("@vchUsuario", value: payload.OrderAdditionalInfo.User);
var terminalParam = new SqlParameter("@vchTerminal", value: payload.OrderAdditionalInfo.Terminal);
var outputMessageParam = new SqlParameter
{
ParameterName = "@vchMensaje",
Value = "",
Direction = ParameterDirection.Output,
SqlDbType = SqlDbType.VarChar,
Size = 255
};
var returnCode = new SqlParameter
{
ParameterName = "@returnCode",
Value = -1,
Direction = ParameterDirection.Output
};
await potigianContext.Database.ExecuteSqlCommandAsync("EXEC @returnCode = dbo.SD03_OC_ING_MERC_ACTU_DETA " +
"@C_PROVEEDOR, @C_OC, @C_SUCU_EMPR, @C_SUCU_DESTINO_ALT, @U_PREFIJO_OC, @U_SUFIJO_OC, @M_OC_MADRE, " +
"@C_SITUAC_OC, @C_ARTICULO, @M_VENDE_POR_PESO, @Q_UNID_KGS_PED, @Q_UNID_KGS_CUMPL, @Q_UNID_KGS_PEND, " +
"@Q_UNID_KGS_ING, @Q_BULTOS_KGS_ING, @Q_FACTOR_PIEZAS_ING, @C_COMP_ING_MERC, " +
"@U_PREFIJO_COMP_ING_MERC, @U_SUFIJO_COMP_ING_MERC, @F_COMP_ING_MERC, @C_PROGRAMA, @D_OBSERVACION, " +
"@vchUsuario, @vchTerminal, @vchMensaje OUTPUT", returnCode, providerParam, ocParam, companyBranchParam,
destinationBranchParam, prefixOcParam, suffixOcParam, ocMotherParam, situationCodeParam, articleCodeParam,
saleByWeightParam, requestedUnitsParam, deliveredUnitsParam, pendingUnitsParam, receivedUnitsParam,
receivedPacksParam, receivedPiecesFactorParam, receivedGoodsBoughtParam, receivedGoodsPrefixParam,
receivedGoodsSuffixParam, goodsDateParam, programCodeParam, observationParam, userParam,
terminalParam, outputMessageParam);
response.DetailInfo.Add(new SynchronizeOrderDetail
{
ArticleCode = orderDetail.ArticleCode,
DetailMessage = (string)outputMessageParam.Value,
DetailReturnCode = (int)returnCode.Value
});
}
potigianContext.Database.CommitTransaction();
return response;
}
catch (SqlException)
{
potigianContext.Database.RollbackTransaction();
throw;
}
}));
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
// Setting Up Log
LogSetup.InitializeLog("Apple_Store_Crawler.log", "info");
_logger = LogManager.GetCurrentClassLogger();
// Starting Flow
_logger.Info("Worker Started");
// Loading Configuration
_logger.Info("Reading Configuration");
LoadConfiguration();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists(fPath))
{
_logger.Fatal("Couldnt find proxies on path : " + fPath);
System.Environment.Exit(-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load(fLines.ToList());
}
catch (Exception ex)
{
_logger.Fatal(ex);
System.Environment.Exit(-101);
}
}
// AWS Queue Handler
_logger.Info("Initializing Queues");
AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret);
// Step 1 - Trying to obtain the root page html (source of all the apps)
var rootPageResponse = httpClient.GetRootPage(shouldUseProxies);
// Sanity Check
if (String.IsNullOrWhiteSpace(rootPageResponse))
{
_logger.Info("Error obtaining Root Page HTMl - Aborting", "Timeout Error");
return;
}
// Step 2 - Extracting Category Urls from the Root Page and queueing their Urls
foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse))
{
// Logging Feedback
_logger.Info("Queueing Category : " + categoryUrl);
// Queueing Category Urls
sqsWrapper.EnqueueMessage(categoryUrl);
}
_logger.Info("End of Bootstrapping phase");
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler();
AppStoreParser parser = new AppStoreParser();
_logger = new LogWrapper();
// Loading Configuration
_logger.LogMessage("Loading Configurations from App.config");
LoadConfiguration();
// AWS Queue Handler
_logger.LogMessage("Initializing Queues");
AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.LogMessage("Started Processing Individual Apps Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsUrlQueue.DeQueueMessages())
{
Thread.Sleep(_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine("Fallback (seconds) => " + waitTime);
Thread.Sleep(waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appUrl in appsUrlQueue.GetDequeuedMessages())
{
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
//appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8";
htmlResponse = httpClient.Get(appUrl.Body);
if (String.IsNullOrEmpty(htmlResponse))
{
// Extending Fallback time
retries++;
int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;
_logger.LogMessage("Retrying Request for App Page [ " + sleepTime / 1000 + " ]", "Request Error", BDC.BDCCommons.TLogEventLevel.Error);
Thread.Sleep(sleepTime);
}
} while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace(htmlResponse))
{
// Deletes Message and moves on
//appsUrlQueue.DeleteMessage (appUrl);
continue;
}
// Feedback
_logger.LogMessage("Current page " + appUrl.Body, "Parsing App Data");
// Parsing Data out of the Html Page
AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse);
parsedApp.url = appUrl.Body;
// Enqueueing App Data
appsDataQueue.EnqueueMessage(parsedApp.ToJson());
// Little Hiccup
Thread.Sleep(_hiccupTime);
}
catch (Exception ex)
{
_logger.LogMessage(ex.Message, "App Url Processing", BDC.BDCCommons.TLogEventLevel.Error);
}
finally
{
//Deleting the message
appsUrlQueue.DeleteMessage(appUrl);
}
}
}
catch (Exception ex)
{
_logger.LogMessage(ex);
}
} while (true);
}
