public async Task <ActionResult <Response <bool> > > ClearAssignedRequestsFromPreparer(int preparer) { logger.LogInformation("POST cabe/asignados/{preparador}/clear invocado con preparador " + preparer); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var assignedRequests = await potigianContext.RequestHeaders .Where(req => (req.PreparerCode == preparer.ToString() || req.PreparerCode == $"CSPRP{preparer}") && req.SituationCode == Config.Requests.StateInPreparation) .ToListAsync(); assignedRequests.ForEach(r => { r.SituationCode = Config.Requests.StateAvailableToPrepare; r.SituationDate = DateTime.Now; r.PreparerCode = null; potigianContext.Update(r); }); await potigianContext.SaveChangesAsync(); return true; })); }
public async Task <ActionResult <Response <List <Preparer> > > > GetPreparers() { logger.LogInformation("GET preparadores invocado"); return(await RequestsHandler.HandleAsyncRequest( logger, async() => await potigianContext.Preparers.ToListAsync())); }
private static void TestAlbumsServicesWithJson(string baseUri, HttpClient client) { IRequestsHandler albumsHandler = new RequestsHandler(baseUri, "albums", client, "application/json"); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get all albums..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(albumsHandler.Get()); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get album by id..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(albumsHandler.GetById(1)); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to add album..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(albumsHandler.Post("{\"title\" : \"Asdf\", \"producerId\" : 1 }")); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to edit album..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(albumsHandler.Put(1, "{\"title\" : \"Asdf\", \"producerId\" : 1 }")); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to delete album..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); albumsHandler.Delete(5); }
public async Task <ActionResult <Response <List <RequestDetails> > > > GetRequestDetails(int prefixDoc, int doc, int suffixDoc) { logger.LogInformation("GET detalle/{prefijo}/{doc}/{sufijo} invocado con prefijo " + prefixDoc + ", doc " + doc + " y sufijo " + suffixDoc); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var requestDetails = await potigianContext.RequestDetails .Where(req => req.DocumentPrefix == prefixDoc && req.DocumentCode == doc && req.DocumentSuffix == suffixDoc) .Join( potigianContext.Articles, req => req.ArticleCode, article => article.Code, (req, article) => req.Append(article)) .ToListAsync(); foreach (var req in requestDetails) { var value = await potigianContext.BranchArticles.FirstOrDefaultAsync(a => a.ArticleCode == req.ArticleCode); if (value.ArticleCode == null || value.ArticleCode == 0) { value.ArticleCode = 1; } req.Append(value); } return requestDetails; })); }
private static void TestSongsServicesWithJson(string baseUri, HttpClient client) { IRequestsHandler songsHandler = new RequestsHandler(baseUri, "songs", client, "application/json"); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get all songs..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(songsHandler.Get()); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get song by id..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(songsHandler.GetById(1)); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to add song..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(songsHandler.Post("{ \"title\": \"Postman\", \"artistId\" : 1, \"albumId\": 1 }")); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to edit song..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(songsHandler.Put(1, "{ \"title\": \"Postman\", \"artistId\" : 1, \"albumId\": 1 }")); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to delete song..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); songsHandler.Delete(5); }
public async Task <ActionResult <Response <List <RequestDetails> > > > GetRequestsDetails() { logger.LogInformation("GET detalle invocado"); return(await RequestsHandler.HandleAsyncRequest( logger, async() => await potigianContext.RequestDetails.ToListAsync())); }
public async Task <ActionResult <Response <List <PurchaseOrderHeader> > > > GetOrdersByProvider(int code) { logger.LogInformation("GET proveedor/{code} invocado con code " + code); return(await RequestsHandler.HandleAsyncRequest( logger, async() => await potigianContext.PurchaseOrdersHeaders.Where(o => o.ProviderCode == code && o.Situation == 1).ToListAsync())); }
private static void TestArtistsServicesWithJson(string baseUri, HttpClient client) { IRequestsHandler artistsHandler = new RequestsHandler(baseUri, "artists", client, "application/json"); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get all artists..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(artistsHandler.Get()); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get artist by id..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(artistsHandler.GetById(1)); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to add artist..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(artistsHandler.Post("{ \"name\" : \"Gosho\" }")); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to edit artist..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(artistsHandler.Put(1, "{ \"name\" : \"Gosho\" }")); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to delete artist..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); artistsHandler.Delete(5); }
public async Task <ActionResult <Response <List <RequestHeaders> > > > GetRequestsHeadersByPreparer(int preparer) { logger.LogInformation("GET cabe/asignados/{preparador} invocado con preparador " + preparer); return(await RequestsHandler.HandleAsyncRequest( logger, async() => await potigianContext.RequestHeaders .Where(h => (h.PreparerCode == preparer.ToString() || h.PreparerCode == $"CSPRP{preparer}") && h.SituationCode == Config.Requests.StateInPreparation) .ToListAsync())); }
public async Task <ActionResult <Response <List <RequestHeaders> > > > GetRequestsHeadersBySituation(int situation) { logger.LogInformation("GET cabe/situacion/{code} invocado con code " + situation); return(await RequestsHandler.HandleAsyncRequest( logger, async() => await potigianContext.RequestHeaders .Where(h => h.SituationCode == situation) .ToListAsync())); }
public async Task <ActionResult <Response <bool> > > StartRequest(int prefixDoc, int doc, int suffixDoc, int preparer) { logger.LogInformation("POST cabe/asignados/{prefijo}/{doc}/{sufijo}/start/{preparador} invocado con prefijo " + prefixDoc + ", doc " + doc + ", sufijo " + suffixDoc + " y preparador " + preparer); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var requestPreparations = potigianContext.RequestPreparations.Where(rp => rp.DocumentSuffix == suffixDoc); if (await requestPreparations.CountAsync() == 0) // new entry { var preparation = new RequestPreparation { Code = preparer.ToString(), DocumentSuffix = suffixDoc, InsertDate = DateTime.Now, StartDate = DateTime.Now, StatusCode = Config.Requests.StateInPreparation, EndDate = null, }; potigianContext.Add(preparation); potigianContext.Add(PreparerProductivity.FromPreparation(preparation)); await potigianContext.SaveChangesAsync(); } else { try { var preparation = await requestPreparations.FirstAsync(); if (preparation.StartDate == null) { preparation.Code = preparer.ToString(); preparation.StartDate = DateTime.Now; preparation.EndDate = null; preparation.StatusCode = Config.Requests.StateInPreparation; potigianContext.Update(preparation); potigianContext.Add(PreparerProductivity.FromPreparation(preparation)); await potigianContext.SaveChangesAsync(); } } catch (Exception ex) { throw ex; } } return true; })); }
private static void TestAlbumsServicesWithXml(string baseUri, HttpClient client) { IRequestsHandler albumsHandler = new RequestsHandler(baseUri, "albums", client, "application/xml"); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get all albums..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(albumsHandler.Get()); Console.WriteLine("\n=================================\n"); Console.WriteLine("Press Enter to get album by id..."); Console.WriteLine("\n=================================\n"); Console.ReadKey(); Console.WriteLine(albumsHandler.GetById(1)); }
public ConnectionManager( ITransportResource transport, CommandBuffer commandBuffer, CommandParser commandParser, RequestsHandler requestsHandler, IDtoSerializer serializer) { _transport = transport; _commandBuffer = commandBuffer; _commandParser = commandParser; _requestsHandler = requestsHandler; _serializer = serializer; _commandBuffer.CommandAssembled += CommandBuffer_CommandAssembled; _transport.DataReceived += data => _commandBuffer.AppendBytes(data); _transport.ConnectionError += Transport_ConnectionError; }
public async Task <ActionResult <Response <Preparer> > > GetPreparer(int id) { logger.LogInformation("GET preparadores/{id} invocado con id " + id); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var preparer = await potigianContext.Preparers.FirstOrDefaultAsync(p => p.Id == id); if (preparer == null) { preparer = new Preparer { Code = "-1" }; } return preparer; })); }
public async Task <ActionResult <Response <List <PurchaseOrderDetails> > > > GetOrderDetails(string prefixCode, string ocCode, string suffixCode) { logger.LogInformation("GET {prefijo}/{oc}/{sufijo} invocado con prefijo " + prefixCode + ", oc " + ocCode + " y sufijo " + suffixCode); return(await RequestsHandler.HandleAsyncRequest( logger, async() => await potigianContext.PurchaseOrderDetails .Where(od => od.PrefixOcCode.ToString() == prefixCode && od.OcCode.ToString() == ocCode && od.SuffixOcCode.ToString() == suffixCode) .Join( potigianContext.Articles, od => od.ArticleCode, article => article.Code, (od, article) => od.Append(article)) .Join( potigianContext.ProviderArticles, od => od.ArticleCode, providerArticle => providerArticle.ArticleCode, (od, providerArticle) => od.Append(providerArticle)) .ToListAsync())); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); _logger = new LogWrapper(); // Loading Configuration _logger.LogMessage("Reading Configuration"); LoadConfiguration(); // AWS Queue Handler _logger.LogMessage("Initializing Queues"); AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret); // Step 1 - Trying to obtain the root page html (source of all the apps) var rootPageResponse = httpClient.GetRootPage(); // Sanity Check if (String.IsNullOrWhiteSpace(rootPageResponse)) { _logger.LogMessage("Error obtaining Root Page HTMl - Aborting", "Timeout Error"); return; } // Step 2 - Extracting Category Urls from the Root Page and queueing their Urls foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse)) { // Logging Feedback _logger.LogMessage("Queueing Category : " + categoryUrl); // Queueing Category Urls sqsWrapper.EnqueueMessage(categoryUrl); } _logger.LogMessage("End of Bootstrapping phase"); }
public async Task <ActionResult <Response <User> > > LoginUser(string usr, string code) { logger.LogInformation($"GET de usuario invocado por " + usr); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var user = await(from u in securityContext.Users join s in securityContext.AccessSystems on u.AccessCode.Trim() equals s.AccessCode.Trim() where s.Name.Trim() == "Mercaderias" && u.Code.Trim() == usr && u.AccessCode.Trim() == code select u).FirstOrDefaultAsync(); if (user == null) { user = new User { Code = "-1" }; } return user; })); }
public async Task <ActionResult <Response <List <PurchaseOrderHeader> > > > GetOrdersByProviderAndSuffixCode(string providerCode, string suffixCode) { logger.LogInformation("GET {sufijo}/proveedor/{code} invocado con sufijo " + suffixCode + " y code " + providerCode); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var expr = default(Expression <Func <PurchaseOrderHeader, bool> >); if (providerCode == "none" && suffixCode == "none") { expr = o => o.Situation == 1; } else if (providerCode == "none") { expr = o => o.OrderSuffix.ToString().Contains(suffixCode) && o.Situation == 1; } else if (suffixCode == "none") { expr = o => o.ProviderCode.ToString().Contains(providerCode) && o.Situation == 1; } else { expr = o => o.OrderSuffix.ToString().Contains(suffixCode) && o.ProviderCode.ToString().Contains(providerCode) && o.Situation == 1; } var ocs = potigianContext.PurchaseOrdersHeaders.Where(expr); await ocs.ForEachAsync(async oc => { oc.Items = await potigianContext.PurchaseOrderDetails .Where(od => od.SuffixOcCode == oc.OrderSuffix && od.PrefixOcCode == oc.OrderPrefix && od.OcCode == oc.OrderCode) .CountAsync(); }); return await ocs.ToListAsync(); })); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); // Loading Configuration LogSetup.InitializeLog("Apple_Store_Numerics_Worker.log", "info"); _logger = LogManager.GetCurrentClassLogger(); // Loading Config _logger.Info("Loading Configurations from App.config"); LoadConfiguration(); // Control Variable (Bool - Should the process use proxies? ) bool shouldUseProxies = false; // Checking for the need to use proxies if (args != null && args.Length == 1) { // Setting flag to true shouldUseProxies = true; // Loading proxies from .txt received as argument String fPath = args[0]; // Sanity Check if (!File.Exists(fPath)) { _logger.Fatal("Couldnt find proxies on path : " + fPath); System.Environment.Exit(-100); } // Reading Proxies from File string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8")); try { // Actual Load of Proxies ProxiesLoader.Load(fLines.ToList()); } catch (Exception ex) { _logger.Fatal(ex); System.Environment.Exit(-101); } } // AWS Queue Handler _logger.Info("Initializing Queues"); AWSSQSHelper numericUrlQueue = new AWSSQSHelper(_numericUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); // Setting Error Flag to No Error ( 0 ) System.Environment.ExitCode = 0; // Initialiazing Control Variables int fallbackWaitTime = 1; _logger.Info("Started Processing Numeric Urls"); do { try { // Dequeueing messages from the Queue if (!numericUrlQueue.DeQueueMessages()) { Thread.Sleep(_hiccupTime); // Hiccup continue; } // Checking for no message received, and false positives situations if (!numericUrlQueue.AnyMessageReceived()) { // If no message was found, increases the wait time int waitTime; if (fallbackWaitTime <= 12) { // Exponential increase on the wait time, truncated after 12 retries waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000); } else // Reseting Wait after 12 fallbacks { waitTime = 2000; fallbackWaitTime = 0; } fallbackWaitTime++; // Sleeping before next try Console.WriteLine("Fallback (seconds) => " + waitTime); Thread.Sleep(waitTime); continue; } // Reseting fallback time fallbackWaitTime = 1; // Iterating over dequeued Messages foreach (var numericUrl in numericUrlQueue.GetDequeuedMessages()) { try { // Retries Counter int retries = 0; string htmlResponse; // Retrying if necessary do { // Executing Http Request for the Category Url htmlResponse = httpClient.Get(numericUrl.Body, shouldUseProxies); if (String.IsNullOrEmpty(htmlResponse)) { _logger.Info("Retrying Request for Category Page"); retries++; } } while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries); // Checking if retries failed if (String.IsNullOrWhiteSpace(htmlResponse)) { // Deletes Message and moves on numericUrlQueue.DeleteMessage(numericUrl); continue; } // Feedback _logger.Info("Current page " + numericUrl.Body); foreach (var parsedAppUrl in parser.ParseAppsUrls(htmlResponse)) { // Enqueueing App Urls appsUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(parsedAppUrl)); } } catch (Exception ex) { _logger.Info(ex); } finally { // Deleting the message numericUrlQueue.DeleteMessage(numericUrl); } } } catch (Exception ex) { _logger.Error(ex); } } while (true); }
public async Task <ActionResult <Response <List <RequestHeaders> > > > AssignRequestsHeadersToPreparer( int preparer, [FromQuery(Name = "cigarrillos")] bool cigarettesOnly, [FromQuery(Name = "pedido")] int suffixNumber, [FromQuery(Name = "reparto")] int deliveryNumber) { logger.LogInformation("POST cabe/asignados/{preparador} invocado con preparador " + preparer); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var assignedRequests = await potigianContext.RequestHeaders .Where(req => (req.PreparerCode == preparer.ToString() || req.PreparerCode == $"CSPRP{preparer}") && req.SituationCode == Config.Requests.StateInPreparation) .ToListAsync(); // If there are any assigned requests, we won't assign new ones. if (assignedRequests.Count > 0) { return assignedRequests; } var newRequests = default(List <RequestHeaders>); if (suffixNumber != 0) { newRequests = await potigianContext.RequestHeaders .Where(req => req.DocumentSuffix == suffixNumber && req.SituationCode == Config.Requests.StateAvailableToPrepare) .ToListAsync(); } else if (deliveryNumber != 0) { newRequests = await potigianContext.RequestHeaders .Where(req => req.DistributionNumber == deliveryNumber && req.SituationCode == Config.Requests.StateAvailableToPrepare) .ToListAsync(); } if (newRequests == default(List <RequestHeaders>) || newRequests.Count == 0) { return new List <RequestHeaders>(); } var suffixes = newRequests.Select(req => req.DocumentSuffix).Distinct(); var prefixes = newRequests.Select(req => req.DocumentPrefix).Distinct(); var docs = newRequests.Select(req => req.DocumentCode).Distinct(); var requestsDetails = await potigianContext.RequestDetails .Where(req => suffixes.Contains(req.DocumentSuffix) && prefixes.Contains(req.DocumentPrefix) && docs.Contains(req.DocumentCode)) .ToListAsync(); var definedRequests = newRequests.Where(req => { var details = requestsDetails .Where(det => req.DocumentSuffix == det.DocumentSuffix && req.DocumentCode == det.DocumentCode && req.DocumentPrefix == det.DocumentPrefix); bool areOnlyCigarettes = details.All(det => new List <decimal?>() { 1, 2 }.Contains(det.FamilyCode)); return cigarettesOnly ? areOnlyCigarettes : !areOnlyCigarettes; }) .Take(config.GetValue <int>("RequestsPerPreparer")) .ToList(); definedRequests.ForEach(req => { req.SituationCode = Config.Requests.StateInPreparation; req.SituationDate = DateTime.Now; req.PreparerCode = preparer.ToString(); potigianContext.Update(req); }); await potigianContext.SaveChangesAsync(); return definedRequests; })); }
public async Task <ActionResult <Response <bool> > > CloseRequestPreparation(int prefixDoc, int doc, int suffixDoc, [FromBody] CloseRequestPayload payload) { logger.LogInformation("POST preparaciones/{prefijo}/{doc}/{sufijo} invocado con prefijo " + prefixDoc + ", doc " + doc + " y sufijo " + suffixDoc); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { bool closedComplete = true; var requestDetails = await potigianContext.RequestDetails .Where(req => req.DocumentPrefix == prefixDoc && req.DocumentCode == doc && req.DocumentSuffix == suffixDoc) .ToListAsync(); var unfinishedRequestDetails = requestDetails .Where(req => req.PackagesGrams != payload.ArticleCount[GetDictionaryCountKey(req)]); var requestHeader = await potigianContext.RequestHeaders .FirstAsync(req => req.DocumentPrefix == prefixDoc && req.DocumentCode == doc && req.DocumentSuffix == suffixDoc); // Unfinished requests case if (unfinishedRequestDetails.Count() > 0) { var missingRequestDetails = unfinishedRequestDetails .Select(req => new RequestMissingDetails(req, payload.ArticleCount[GetDictionaryCountKey(req)])) .ToList(); foreach (var request in unfinishedRequestDetails) { int count = payload.ArticleCount[GetDictionaryCountKey(request)]; if (count != 0) { request.PackagesGrams = count; request.ArticleTotal = request.PackagesGrams *request.ArticleUnitaryPrice; potigianContext.RequestDetails.Update(request); } } potigianContext.RequestMissingDetails.AddRange(missingRequestDetails); closedComplete = false; } var preparation = await potigianContext.RequestPreparations.FirstAsync(p => p.DocumentSuffix == suffixDoc); preparation.EndDate = DateTime.Now; preparation.StatusCode = Config.Requests.StateClosed; preparation.MovementFlag = 'E'; var productivity = await potigianContext.PreparerProductivities.FirstAsync(p => p.DocumentSuffix == suffixDoc); productivity.EndDate = preparation.EndDate; productivity.StatusCode = preparation.StatusCode; productivity.MovementFlag = preparation.MovementFlag; var productivityDifferential = productivity.EndDate.Value.Subtract(productivity.StartDate.Value); productivity.Days = productivityDifferential.Days; productivity.Hours = productivityDifferential.Hours; productivity.Minutes = productivityDifferential.Minutes; productivity.Seconds = productivityDifferential.Seconds; requestHeader.SituationCode = Config.Requests.StateClosed; requestHeader.SituationDate = DateTime.Now; requestHeader.Printer = payload.Printer; requestHeader.TotalPackages = payload.Bags; potigianContext.RequestHeaders.Update(requestHeader); potigianContext.RequestPreparations.Update(preparation); potigianContext.PreparerProductivities.Update(productivity); await potigianContext.SaveChangesAsync(); return closedComplete; })); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); // Loading Configuration LogSetup.InitializeLog("Apple_Store_Urls_Worker.log", "info"); _logger = LogManager.GetCurrentClassLogger(); // Loading Config _logger.Info("Loading Configurations from App.config"); LoadConfiguration(); // Control Variable (Bool - Should the process use proxies? ) bool shouldUseProxies = false; // Checking for the need to use proxies if (args != null && args.Length == 1) { // Setting flag to true shouldUseProxies = true; // Loading proxies from .txt received as argument String fPath = args[0]; // Sanity Check if (!File.Exists(fPath)) { _logger.Fatal("Couldnt find proxies on path : " + fPath); System.Environment.Exit(-100); } // Reading Proxies from File string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8")); try { // Actual Load of Proxies ProxiesLoader.Load(fLines.ToList()); } catch (Exception ex) { _logger.Fatal(ex); System.Environment.Exit(-101); } } // AWS Queue Handler _logger.Info("Initializing Queues"); AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); // Setting Error Flag to No Error ( 0 ) System.Environment.ExitCode = 0; // Initialiazing Control Variables int fallbackWaitTime = 1; _logger.Info("Started Processing Individual Apps Urls"); do { try { // Dequeueing messages from the Queue if (!appsUrlQueue.DeQueueMessages()) { Thread.Sleep(_hiccupTime); // Hiccup continue; } // Checking for no message received, and false positives situations if (!appsUrlQueue.AnyMessageReceived()) { // If no message was found, increases the wait time int waitTime; if (fallbackWaitTime <= 12) { // Exponential increase on the wait time, truncated after 12 retries waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000); } else // Reseting Wait after 12 fallbacks { waitTime = 2000; fallbackWaitTime = 0; } fallbackWaitTime++; // Sleeping before next try Console.WriteLine("Fallback (seconds) => " + waitTime); Thread.Sleep(waitTime); continue; } // Reseting fallback time fallbackWaitTime = 1; // Iterating over dequeued Messages foreach (var appUrl in appsUrlQueue.GetDequeuedMessages()) { bool processingWorked = true; try { // Retries Counter int retries = 0; string htmlResponse; // Retrying if necessary do { // Executing Http Request for the Category Url //appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8"; //appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8"; //appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8"; //appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8"; htmlResponse = httpClient.Get(appUrl.Body, shouldUseProxies); if (String.IsNullOrEmpty(htmlResponse)) { // Extending Fallback time retries++; int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000; _logger.Info("Retrying Request for App Page [ " + sleepTime / 1000 + " ]"); Thread.Sleep(sleepTime); } } while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries); // Checking if retries failed if (String.IsNullOrWhiteSpace(htmlResponse)) { continue; } // Feedback _logger.Info("Current page " + appUrl.Body, "Parsing App Data"); // Parsing Data out of the Html Page AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse); parsedApp.url = appUrl.Body; // Enqueueing App Data appsDataQueue.EnqueueMessage(parsedApp.ToJson()); // Little Hiccup Thread.Sleep(_hiccupTime); } catch (Exception ex) { _logger.Error(ex); // Setting Flag to "False" processingWorked = false; } finally { //Deleting the message - Only if the processing worked if (processingWorked) { appsUrlQueue.DeleteMessage(appUrl); } } } } catch (Exception ex) { _logger.Error(ex); } } while (true); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); _logger = new LogWrapper(); // Loading Configuration _logger.LogMessage("Loading Configurations from App.config"); LoadConfiguration(); // AWS Queue Handler _logger.LogMessage("Initializing Queues"); AWSSQSHelper categoriesUrlQueue = new AWSSQSHelper(_categoriesQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); AWSSQSHelper charactersUrlQueue = new AWSSQSHelper(_characterUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); // Setting Error Flag to No Error ( 0 ) System.Environment.ExitCode = 0; // Initialiazing Control Variables int fallbackWaitTime = 1; _logger.LogMessage("Started Processing Category Urls"); do { try { // Dequeueing messages from the Queue if (!categoriesUrlQueue.DeQueueMessages()) { Thread.Sleep(_hiccupTime); // Hiccup continue; } // Checking for no message received, and false positives situations if (!categoriesUrlQueue.AnyMessageReceived()) { // If no message was found, increases the wait time int waitTime; if (fallbackWaitTime <= 12) { // Exponential increase on the wait time, truncated after 12 retries waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000); } else // Reseting Wait after 12 fallbacks { waitTime = 2000; fallbackWaitTime = 0; } fallbackWaitTime++; // Sleeping before next try Console.WriteLine("Fallback (seconds) => " + waitTime); Thread.Sleep(waitTime); continue; } // Reseting fallback time fallbackWaitTime = 1; // Iterating over dequeued Messages foreach (var categoryUrl in categoriesUrlQueue.GetDequeuedMessages()) { // Console Feedback _logger.LogMessage("Started Parsing Category : " + categoryUrl.Body); try { // Retries Counter int retries = 0; string htmlResponse; // Retrying if necessary do { // Executing Http Request for the Category Url htmlResponse = httpClient.Get(categoryUrl.Body); if (String.IsNullOrEmpty(htmlResponse)) { _logger.LogMessage("Retrying Request for Category Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error); retries++; } } while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries); // Checking if retries failed if (String.IsNullOrWhiteSpace(htmlResponse)) { // Deletes Message and moves on categoriesUrlQueue.DeleteMessage(categoryUrl); continue; } // If the request worked, parses the urls out of the page foreach (string characterUrls in parser.ParseCharacterUrls(htmlResponse)) { // Enqueueing Urls charactersUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(characterUrls)); } } catch (Exception ex) { _logger.LogMessage(ex.Message, "Category Url Processing", BDC.BDCCommons.TLogEventLevel.Error); } finally { // Deleting the message categoriesUrlQueue.DeleteMessage(categoryUrl); } } } catch (Exception ex) { _logger.LogMessage(ex); } } while (true); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); _logger = new LogWrapper(); // Loading Configuration _logger.LogMessage("Loading Configurations from App.config"); LoadConfiguration(); // AWS Queue Handler _logger.LogMessage("Initializing Queues"); AWSSQSHelper charactersUrlQueue = new AWSSQSHelper(_characterUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); AWSSQSHelper numericUrlQueue = new AWSSQSHelper(_numericUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); // Setting Error Flag to No Error ( 0 ) System.Environment.ExitCode = 0; // Initialiazing Control Variables int fallbackWaitTime = 1; _logger.LogMessage("Started Processing Character Urls"); do { try { // Dequeueing messages from the Queue if (!charactersUrlQueue.DeQueueMessages()) { Thread.Sleep(_hiccupTime); // Hiccup continue; } // Checking for no message received, and false positives situations if (!charactersUrlQueue.AnyMessageReceived()) { // If no message was found, increases the wait time int waitTime; if (fallbackWaitTime <= 12) { // Exponential increase on the wait time, truncated after 12 retries waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000); } else // Reseting Wait after 12 fallbacks { waitTime = 2000; fallbackWaitTime = 0; } fallbackWaitTime++; // Sleeping before next try Console.WriteLine("Fallback (seconds) => " + waitTime); Thread.Sleep(waitTime); continue; } // Reseting fallback time fallbackWaitTime = 1; // Iterating over dequeued Messages foreach (var characterUrl in charactersUrlQueue.GetDequeuedMessages()) { // Console Feedback _logger.LogMessage("Started Parsing Url : " + characterUrl.Body); try { // Retries Counter int retries = 0; string htmlResponse; // Retrying if necessary do { // Executing Http Request for the Category Url htmlResponse = httpClient.Get(characterUrl.Body); if (String.IsNullOrEmpty(htmlResponse)) { _logger.LogMessage("Retrying Request for Character Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error); retries++; // Small Hiccup Thread.Sleep(_hiccupTime); } } while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries); // Checking if retries failed if (String.IsNullOrWhiteSpace(htmlResponse)) { // Deletes Message and moves on charactersUrlQueue.DeleteMessage(characterUrl); continue; } // Hashset of urls processed (to avoid duplicates) HashSet <String> urlsQueued = new HashSet <String> (); // Executing Request and Queueing Urls until there's no other Url to be queued do { // Flag to check whether any url was added after the last iteration (avoids endless loop) bool anyNewUrl = false; // If the request worked, parses the Urls out of the page foreach (string numericUrls in parser.ParseNumericUrls(htmlResponse).Select(t => HttpUtility.HtmlDecode(t))) { // Checking if this url was previously queued if (!urlsQueued.Contains(numericUrls)) { // Enqueueing Urls numericUrlQueue.EnqueueMessage(HttpUtility.HtmlDecode(numericUrls)); // Adding url to the local hashset urlsQueued.Add(numericUrls); anyNewUrl = true; } } // Checking for the need to perform another http request for the next page if (parser.IsLastPage(htmlResponse) || !anyNewUrl) { break; // Breaks "While" Loop } // Feedback _logger.LogMessage("Urls Queued For This Page : " + urlsQueued.Count, "\n\tProcessing Feedback"); // If it got to this point, it means that there are more pages to be processed // Parsing URL of the "Last" page (the last that's visible) string lastPageUrl = HttpUtility.HtmlDecode(parser.ParseLastPageUrl(htmlResponse)); // Executing Http Request for this Url (with retries) retries = 0; do { // HTTP Get for the Page htmlResponse = httpClient.Get(lastPageUrl); if (String.IsNullOrEmpty(htmlResponse)) { _logger.LogMessage("Retrying Request for Last Page", "Request Error", BDC.BDCCommons.TLogEventLevel.Error); retries++; // Small Hiccup Thread.Sleep(_hiccupTime); } } while (String.IsNullOrEmpty(htmlResponse) && retries <= _maxRetries); } while (true); } catch (Exception ex) { _logger.LogMessage(ex); } finally { charactersUrlQueue.DeleteMessage(characterUrl); } } } catch (Exception ex) { _logger.LogMessage(ex); } } while (true); }
public async Task <ActionResult <Response <SynchronizeOrderResponse> > > SynchronizeOrder(string prefixCode, string ocCode, string suffixCode, [FromBody] SynchronizeOrderPayload payload) { logger.LogInformation("POST {prefijo}/{oc}/{sufijo} invocado con prefijo " + prefixCode + ", oc " + ocCode + " y sufijo " + suffixCode); return(await RequestsHandler.HandleAsyncRequest( logger, async() => { var orderHeader = await potigianContext.PurchaseOrdersHeaders .FirstAsync(h => h.OrderPrefix == decimal.Parse(prefixCode) && h.OrderSuffix == decimal.Parse(suffixCode) && h.OrderCode == decimal.Parse(ocCode)); var orderDetails = await potigianContext.PurchaseOrderDetails .Where(d => d.PrefixOcCode == decimal.Parse(prefixCode) && d.SuffixOcCode == decimal.Parse(suffixCode) && d.OcCode == decimal.Parse(ocCode)) .ToListAsync(); var articleCodes = orderDetails.Select(d => d.ArticleCode); var articles = await potigianContext.Articles.Where(a => articleCodes.Contains(a.Code)).ToListAsync(); var detailsPending = orderDetails.Where(d => d.RequestedPackages > payload.ArticleCount[d.ArticleCode]); await potigianContext.Database.BeginTransactionAsync(); var response = new SynchronizeOrderResponse(); try { // CABE Procedure { var ocParam = new SqlParameter("@C_OC", value: decimal.Parse(ocCode)); var prefixParam = new SqlParameter("@U_PREFIJO_OC", value: decimal.Parse(prefixCode)); var suffixParam = new SqlParameter("@U_SUFIJO_OC", value: decimal.Parse(suffixCode)); var situationParam = new SqlParameter("@C_SITUAC_OC", value: (decimal)(detailsPending.Any() ? 1 : 2)); // 1 pending - 2 ready var receivedGoodsDateParam = new SqlParameter("@f_comp_ing_merc", value: payload.OrderAdditionalInfo.Date); var receivedGoodsCodeParam = new SqlParameter("@C_COMP_ING_MERC", value: (decimal)payload.OrderAdditionalInfo.BillType); var purchasePrefixParam = new SqlParameter("@U_PREFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.PrefixOc); var purchaseSuffixParam = new SqlParameter("@U_SUFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.SuffixOc); var observationParam = new SqlParameter("@D_OBSERVACION_ING_MERC", value: payload.OrderAdditionalInfo.Observations); var userParam = new SqlParameter("@vchUsuario", value: payload.OrderAdditionalInfo.User); var terminalParam = new SqlParameter("@vchTerminal", value: payload.OrderAdditionalInfo.Terminal); var outputMessageParam = new SqlParameter { ParameterName = "@vchMensaje", Value = "", Direction = ParameterDirection.Output, SqlDbType = SqlDbType.VarChar, Size = 255 }; var returnCode = new SqlParameter { ParameterName = "@returnCode", Value = -1, Direction = ParameterDirection.Output, }; await potigianContext.Database.ExecuteSqlCommandAsync("EXEC @returnCode = dbo.SD03_OC_ING_MERC_ACTU_CABE " + "@C_OC, @U_PREFIJO_OC, @U_SUFIJO_OC, @C_SITUAC_OC, @f_comp_ing_merc, @C_COMP_ING_MERC, " + "@U_PREFIJO_COMP_ING_MERC, @U_SUFIJO_COMP_ING_MERC, @D_OBSERVACION_ING_MERC, " + "@vchUsuario, @vchTerminal, @vchMensaje OUTPUT", returnCode, ocParam, prefixParam, suffixParam, situationParam, receivedGoodsDateParam, receivedGoodsCodeParam, purchasePrefixParam, purchaseSuffixParam, observationParam, userParam, terminalParam, outputMessageParam); response.HeaderMessage = (string)outputMessageParam.Value; response.HeaderReturnCode = (int)returnCode.Value; } // DETA PROCEDURE foreach (var orderDetail in orderDetails) { var providerParam = new SqlParameter("@C_PROVEEDOR", value: orderHeader.ProviderCode); var ocParam = new SqlParameter("@C_OC", value: decimal.Parse(ocCode)); var companyBranchParam = new SqlParameter("@C_SUCU_EMPR", value: payload.OrderAdditionalInfo.Branch); var destinationBranchParam = new SqlParameter("@C_SUCU_DESTINO_ALT", value: orderHeader.AlternativeDestinationBranch); var prefixOcParam = new SqlParameter("@U_PREFIJO_OC", value: decimal.Parse(prefixCode)); var suffixOcParam = new SqlParameter("@U_SUFIJO_OC", value: decimal.Parse(suffixCode)); var ocMotherParam = new SqlParameter("@M_OC_MADRE", value: orderHeader.MotherFlag); var situationCodeParam = new SqlParameter("@C_SITUAC_OC", value: (decimal)(detailsPending.Any(d => d.ArticleCode == orderDetail.ArticleCode) ? 1 : 2)); // 1 pending - 2 ready var articleCodeParam = new SqlParameter("@C_ARTICULO", value: orderDetail.ArticleCode); var saleByWeightParam = new SqlParameter("@M_VENDE_POR_PESO", value: articles.First(a => a.Code == orderDetail.ArticleCode).SellByWeight); var requestedUnitsParam = new SqlParameter("@Q_UNID_KGS_PED", value: orderDetail.RequestedPackages); var deliveredUnitsParam = new SqlParameter("@Q_UNID_KGS_CUMPL", value: payload.ArticleCount[orderDetail.ArticleCode]); var pendingUnitsParam = new SqlParameter("@Q_UNID_KGS_PEND", value: orderDetail.RequestedPackages - payload.ArticleCount[orderDetail.ArticleCode]); var receivedUnitsParam = new SqlParameter("@Q_UNID_KGS_ING", value: orderDetail.RequestedPackages); var receivedPacksParam = new SqlParameter("@Q_BULTOS_KGS_ING", value: orderDetail.RequestedBulks); var receivedPiecesFactorParam = new SqlParameter("@Q_FACTOR_PIEZAS_ING", value: orderDetail.RequestedFactor); var receivedGoodsBoughtParam = new SqlParameter("@C_COMP_ING_MERC", value: payload.OrderAdditionalInfo.BillType); var receivedGoodsPrefixParam = new SqlParameter("@U_PREFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.PrefixOc); var receivedGoodsSuffixParam = new SqlParameter("@U_SUFIJO_COMP_ING_MERC", value: payload.OrderAdditionalInfo.SuffixOc); var goodsDateParam = new SqlParameter("@F_COMP_ING_MERC", value: payload.OrderAdditionalInfo.Date); var programCodeParam = new SqlParameter("@C_PROGRAMA", value: Config.ProgramCode); var observationParam = new SqlParameter("@D_OBSERVACION", value: payload.OrderAdditionalInfo.Observations); var userParam = new SqlParameter("@vchUsuario", value: payload.OrderAdditionalInfo.User); var terminalParam = new SqlParameter("@vchTerminal", value: payload.OrderAdditionalInfo.Terminal); var outputMessageParam = new SqlParameter { ParameterName = "@vchMensaje", Value = "", Direction = ParameterDirection.Output, SqlDbType = SqlDbType.VarChar, Size = 255 }; var returnCode = new SqlParameter { ParameterName = "@returnCode", Value = -1, Direction = ParameterDirection.Output }; await potigianContext.Database.ExecuteSqlCommandAsync("EXEC @returnCode = dbo.SD03_OC_ING_MERC_ACTU_DETA " + "@C_PROVEEDOR, @C_OC, @C_SUCU_EMPR, @C_SUCU_DESTINO_ALT, @U_PREFIJO_OC, @U_SUFIJO_OC, @M_OC_MADRE, " + "@C_SITUAC_OC, @C_ARTICULO, @M_VENDE_POR_PESO, @Q_UNID_KGS_PED, @Q_UNID_KGS_CUMPL, @Q_UNID_KGS_PEND, " + "@Q_UNID_KGS_ING, @Q_BULTOS_KGS_ING, @Q_FACTOR_PIEZAS_ING, @C_COMP_ING_MERC, " + "@U_PREFIJO_COMP_ING_MERC, @U_SUFIJO_COMP_ING_MERC, @F_COMP_ING_MERC, @C_PROGRAMA, @D_OBSERVACION, " + "@vchUsuario, @vchTerminal, @vchMensaje OUTPUT", returnCode, providerParam, ocParam, companyBranchParam, destinationBranchParam, prefixOcParam, suffixOcParam, ocMotherParam, situationCodeParam, articleCodeParam, saleByWeightParam, requestedUnitsParam, deliveredUnitsParam, pendingUnitsParam, receivedUnitsParam, receivedPacksParam, receivedPiecesFactorParam, receivedGoodsBoughtParam, receivedGoodsPrefixParam, receivedGoodsSuffixParam, goodsDateParam, programCodeParam, observationParam, userParam, terminalParam, outputMessageParam); response.DetailInfo.Add(new SynchronizeOrderDetail { ArticleCode = orderDetail.ArticleCode, DetailMessage = (string)outputMessageParam.Value, DetailReturnCode = (int)returnCode.Value }); } potigianContext.Database.CommitTransaction(); return response; } catch (SqlException) { potigianContext.Database.RollbackTransaction(); throw; } })); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); // Setting Up Log LogSetup.InitializeLog("Apple_Store_Crawler.log", "info"); _logger = LogManager.GetCurrentClassLogger(); // Starting Flow _logger.Info("Worker Started"); // Loading Configuration _logger.Info("Reading Configuration"); LoadConfiguration(); // Control Variable (Bool - Should the process use proxies? ) bool shouldUseProxies = false; // Checking for the need to use proxies if (args != null && args.Length == 1) { // Setting flag to true shouldUseProxies = true; // Loading proxies from .txt received as argument String fPath = args[0]; // Sanity Check if (!File.Exists(fPath)) { _logger.Fatal("Couldnt find proxies on path : " + fPath); System.Environment.Exit(-100); } // Reading Proxies from File string[] fLines = File.ReadAllLines(fPath, Encoding.GetEncoding("UTF-8")); try { // Actual Load of Proxies ProxiesLoader.Load(fLines.ToList()); } catch (Exception ex) { _logger.Fatal(ex); System.Environment.Exit(-101); } } // AWS Queue Handler _logger.Info("Initializing Queues"); AWSSQSHelper sqsWrapper = new AWSSQSHelper(_categoriesQueueName, 10, _awsKey, _awsKeySecret); // Step 1 - Trying to obtain the root page html (source of all the apps) var rootPageResponse = httpClient.GetRootPage(shouldUseProxies); // Sanity Check if (String.IsNullOrWhiteSpace(rootPageResponse)) { _logger.Info("Error obtaining Root Page HTMl - Aborting", "Timeout Error"); return; } // Step 2 - Extracting Category Urls from the Root Page and queueing their Urls foreach (var categoryUrl in parser.ParseCategoryUrls(rootPageResponse)) { // Logging Feedback _logger.Info("Queueing Category : " + categoryUrl); // Queueing Category Urls sqsWrapper.EnqueueMessage(categoryUrl); } _logger.Info("End of Bootstrapping phase"); }
static void Main(string[] args) { // Creating Needed Instances RequestsHandler httpClient = new RequestsHandler(); AppStoreParser parser = new AppStoreParser(); _logger = new LogWrapper(); // Loading Configuration _logger.LogMessage("Loading Configurations from App.config"); LoadConfiguration(); // AWS Queue Handler _logger.LogMessage("Initializing Queues"); AWSSQSHelper appsUrlQueue = new AWSSQSHelper(_appUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); AWSSQSHelper appsDataQueue = new AWSSQSHelper(_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret); // Setting Error Flag to No Error ( 0 ) System.Environment.ExitCode = 0; // Initialiazing Control Variables int fallbackWaitTime = 1; _logger.LogMessage("Started Processing Individual Apps Urls"); do { try { // Dequeueing messages from the Queue if (!appsUrlQueue.DeQueueMessages()) { Thread.Sleep(_hiccupTime); // Hiccup continue; } // Checking for no message received, and false positives situations if (!appsUrlQueue.AnyMessageReceived()) { // If no message was found, increases the wait time int waitTime; if (fallbackWaitTime <= 12) { // Exponential increase on the wait time, truncated after 12 retries waitTime = Convert.ToInt32(Math.Pow(2, fallbackWaitTime) * 1000); } else // Reseting Wait after 12 fallbacks { waitTime = 2000; fallbackWaitTime = 0; } fallbackWaitTime++; // Sleeping before next try Console.WriteLine("Fallback (seconds) => " + waitTime); Thread.Sleep(waitTime); continue; } // Reseting fallback time fallbackWaitTime = 1; // Iterating over dequeued Messages foreach (var appUrl in appsUrlQueue.GetDequeuedMessages()) { try { // Retries Counter int retries = 0; string htmlResponse; // Retrying if necessary do { // Executing Http Request for the Category Url //appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8"; //appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8"; //appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8"; //appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8"; htmlResponse = httpClient.Get(appUrl.Body); if (String.IsNullOrEmpty(htmlResponse)) { // Extending Fallback time retries++; int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000; _logger.LogMessage("Retrying Request for App Page [ " + sleepTime / 1000 + " ]", "Request Error", BDC.BDCCommons.TLogEventLevel.Error); Thread.Sleep(sleepTime); } } while (String.IsNullOrWhiteSpace(htmlResponse) && retries <= _maxRetries); // Checking if retries failed if (String.IsNullOrWhiteSpace(htmlResponse)) { // Deletes Message and moves on //appsUrlQueue.DeleteMessage (appUrl); continue; } // Feedback _logger.LogMessage("Current page " + appUrl.Body, "Parsing App Data"); // Parsing Data out of the Html Page AppleStoreAppModel parsedApp = parser.ParseAppPage(htmlResponse); parsedApp.url = appUrl.Body; // Enqueueing App Data appsDataQueue.EnqueueMessage(parsedApp.ToJson()); // Little Hiccup Thread.Sleep(_hiccupTime); } catch (Exception ex) { _logger.LogMessage(ex.Message, "App Url Processing", BDC.BDCCommons.TLogEventLevel.Error); } finally { //Deleting the message appsUrlQueue.DeleteMessage(appUrl); } } } catch (Exception ex) { _logger.LogMessage(ex); } } while (true); }