private async Task <Cookie> ReadAntiScrappingCookieAsync() { Cookie antiScrappingCookie = null; Log.Information("Initializing Anti-Scrapping Cookie."); try { var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(60) }; httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue { NoCache = true }; httpClient.DefaultRequestHeaders.TryAddWithoutValidation( "user-agent", ScraperConfigurations.GetSupportedAgent()); var requester = new HttpClientRequester(httpClient); var config = Configuration.Default.WithRequester(requester) .WithDefaultLoader(new LoaderOptions { IsResourceLoadingEnabled = true }).WithJs(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(ScraperConfigurations.StoresJsonUrl).WaitUntilAvailable(); var content = document.Body.TextContent; var match = Regex.Match(content, @"Server\sError\s+406"); if (!match.Success && !string.IsNullOrWhiteSpace(content)) { var parametersMatch = this.RegexCall.Match(content); if (parametersMatch.Success) { var cookieName = parametersMatch.Groups[1].Value.Trim(); var toNumbersACall = this.RegexA.Match(content).Groups[1].Value; var toNumbersBCall = this.RegexB.Match(content).Groups[1].Value; var toNumbersCCall = this.RegexC.Match(content).Groups[1].Value; var parameters = parametersMatch.Groups[2].Value; parameters = parameters.Replace("a", "%A%").Replace("b", "%B%").Replace("c", "%C%"); parameters = parameters.Replace("%A%", toNumbersACall).Replace("%B%", toNumbersBCall) .Replace("%C%", toNumbersCCall); // Review: looks like the WaitUntilAvailable method is not working properly. var cookieValue = string.Empty; while (string.IsNullOrWhiteSpace(cookieValue)) { try { cookieValue = document.ExecuteScript($"toHex(slowAES.decrypt({parameters}))") .ToString(); } catch (Exception e) { Log.Warning(e, "Error retrieving the Anti-Scrapping cookie."); await Task.Delay(100); } } Log.Information( "Read cookie '{CookieName}' with value '{CookieValue}'.", cookieName, cookieValue); antiScrappingCookie = new Cookie(cookieName, cookieValue, "/", "www.tuenvio.cu"); } } } catch (Exception e) { Log.Warning(e, "Error evaluating the Anti-Scrapping cookie."); } return(antiScrappingCookie); }
// This method gets called by the runtime. Use this method to add services to the container. // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { services.AddControllersWithViews(); services.AddRazorPages(); services.AddSignalR(); // TODO: Change the ServiceLifetime. The usage of ServiceLifetime.Transient is because multiple threads operations are running in the same dbcontext. services.AddDbContext <DbContext, ApplicationDbContext>(ServiceLifetime.Transient); services.AddOrcEntityFrameworkCore(); services.AddDatabaseSeeder <ApplicationDbSeeder>(); var token = this.Configuration.GetSection("TelegramBot")?["Token"]; if (!string.IsNullOrWhiteSpace(token)) { if (token == "%TELEGRAM_BOT_TOKEN%") { Log.Warning( "Telegram notification is disable. Replace %TELEGRAM_BOT_TOKEN% placeholder in the configuration file with a valid bot token."); } else { Log.Information("Telegram notification is enable."); services.AddTransient <ITelegramBotClient>( sp => new TelegramBotClient(token)); services.AddSingleton <ITelegramCommander, TelegramCommander>(); } } else { Log.Warning( "Telegram notification is disable. To enable it, add a TelegramBot section with a key Token."); } HttpClientExtensions.Configure(this.Configuration); services.AddTransient(sp => new CookieContainer()); services.AddTransient(sp => BrowsingContext.New(AngleSharp.Configuration.Default)); services.AddTransient( sp => { var cookieContainer = sp.GetService <CookieContainer>(); var handler = new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli, AllowAutoRedirect = true }; if (cookieContainer != null) { handler.CookieContainer = cookieContainer; } var httpTimeoutInSeconds = this.Configuration.GetSection("Http")?["TimeoutInSeconds"]; var httpClient = new HttpClient(handler) { Timeout = float.TryParse( httpTimeoutInSeconds, out var timeoutInSeconds) ? TimeSpan.FromSeconds(timeoutInSeconds) : ScraperConfigurations.HttpClientTimeout }; httpClient.DefaultRequestHeaders.TryAddWithoutValidation( "user-agent", ScraperConfigurations.GetSupportedAgent()); httpClient.DefaultRequestHeaders.TryAddWithoutValidation( "accept-encoding", "gzip, deflate, br"); httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue { NoCache = true }; return(httpClient); }); services.AddScoped <IStoreService, StoreService>(); services.AddSingleton <ICacheStorage <string, Product> >( provider => new CacheStorage <string, Product>(storeNullValues: true)); services.AddSingleton <ICacheStorage <string, Department> >( provider => new CacheStorage <string, Department>(storeNullValues: true)); services.AddSingleton <ICacheStorage <string, Store> >( provider => new CacheStorage <string, Store>(storeNullValues: true)); services.AddSingleton <ICookiesAwareHttpClientFactory, CookiesAwareHttpClientFactory>(); services.AddSingleton <IOfficialStoreInfoService, OfficialStoreInfoService>(); services.AddTransient <IEntityScraper <Product>, ProductScraper>(); services.AddTransient <IEntityScraper <Department>, DepartmentScraper>(); services.AddTransient <IEntityScraper <Store>, StoreScraper>(); services.AddTransient <IMultiEntityScraper <Product>, InspectDepartmentProductsScraper>(); services.AddTransient <IMultiEntityScraper <Department>, InspectStoreDepartmentsScraper>(); services.AddSingleton <ImportStoresHostedService>(); services.AddHostedService <AuthenticationHostedService>(); services.AddHostedService <DepartmentMonitorHostedService>(); // services.AddHostedService<ProductMonitorHostedService>(); services.AddHostedService <StoreMonitorHostedService>(); services.AddHostedService <CookieSerializationHostedService>(); // services.AddHostedService<SyncUsersFromTelegramHostedService>(); }
private async Task <Dictionary <string, Cookie> > LoginAsync( Cookie antiScrappingCookie, string url, string username, string password, bool unattended) { // TODO: Improve this. var storeSlug = UriHelper.GetStoreSlug(url); var storeCaptchaFilePath = $"captchas/{storeSlug}.jpg"; var storeCaptchaSolutionFilePath = $"captchas/{storeSlug}.txt"; Log.Information("Authenticating in TuEnvio as {username}", username); var signInUrl = url.Replace("/Products?depPid=0", "/signin.aspx"); var captchaUrl = url.Replace("/Products?depPid=0", "/captcha.ashx"); var cookieContainer = new CookieContainer(); string captchaFilePath = null; var captchaText = string.Empty; var isAuthenticated = false; var attempts = 0; CookieCollection httpHandlerCookieCollection = null; do { attempts++; var httpMessageHandler = new HttpClientHandler { CookieContainer = cookieContainer }; if (antiScrappingCookie != null) { cookieContainer.Add(ScraperConfigurations.CookieCollectionUrl, antiScrappingCookie); } var httpClient = new HttpClient(httpMessageHandler) { Timeout = ScraperConfigurations.HttpClientTimeout }; httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue { NoCache = true }; httpClient.DefaultRequestHeaders.TryAddWithoutValidation( "user-agent", ScraperConfigurations.GetSupportedAgent()); var browsingContext = BrowsingContext.New(Configuration.Default.WithJs()); IDocument signinPageDocument = null; try { var httpResponseMessage = await httpClient.GetCaptchaSaveAsync(signInUrl); if (httpResponseMessage?.Content != null) { var signinPageContent = await httpResponseMessage.Content.ReadAsStringAsync(); signinPageDocument = await browsingContext.OpenAsync(req => req.Content(signinPageContent)); } } catch (Exception e) { Log.Warning(e, "Error retrieving sign in page with url '{Url}'", signInUrl); } Dictionary <string, string> signInParameters = null; try { signInParameters = await BuildSignInParametersAsync(username, password, signinPageDocument); } catch (Exception e) { Log.Warning(e, "Error building sign in parameters for '{Url}'", signInUrl); } if (signInParameters != null) { var captchaImg = signinPageDocument.QuerySelector <IElement>("#cphPage_Login_captch"); var captchaImgSrc = captchaImg?.Attributes["src"]?.Value; if (!string.IsNullOrWhiteSpace(captchaImgSrc)) { captchaUrl = captchaUrl.Replace("captcha.ashx", captchaImgSrc); } captchaFilePath = await DownloadCaptchaAsync(httpClient, captchaUrl); if (!string.IsNullOrWhiteSpace(captchaFilePath) && File.Exists(captchaFilePath)) { if (unattended) { captchaText = GetCaptchaText(captchaFilePath); } else { File.Delete(storeCaptchaSolutionFilePath); File.Copy(captchaFilePath, storeCaptchaFilePath, true); while (File.Exists(storeCaptchaFilePath) && !File.Exists(storeCaptchaSolutionFilePath)) { await Task.Delay(1000); } captchaText = await File.ReadAllTextAsync(storeCaptchaSolutionFilePath); File.Delete(storeCaptchaFilePath); File.Delete(storeCaptchaSolutionFilePath); } if (!string.IsNullOrWhiteSpace(captchaText)) { signInParameters.Add("ctl00$cphPage$Login$capcha", captchaText); try { await httpClient.PostAsync(signInUrl, new FormUrlEncodedContent(signInParameters)); httpHandlerCookieCollection = cookieContainer.GetCookies(ScraperConfigurations.CookieCollectionUrl); isAuthenticated = !string.IsNullOrWhiteSpace(httpHandlerCookieCollection["ShopMSAuth"]?.Value); } catch (Exception e) { Log.Warning(e, "Error authenticating in '{Url}'", signInUrl); } } } try { if (!isAuthenticated) { File.Delete(captchaFilePath); } } catch (Exception e) { Log.Warning(e, "Error deleting captcha file '{FilePath}'", captchaFilePath); } } }while (attempts < 5 && !isAuthenticated); if (isAuthenticated) { try { File.Move(captchaFilePath, $"captchas/{captchaText}.jpg", true); } catch (Exception e) { Log.Warning(e, "Error moving captcha file {FilePath}", captchaFilePath); } } var cookiesCollection = new Dictionary <string, Cookie>(); if (httpHandlerCookieCollection != null) { foreach (Cookie cookie in httpHandlerCookieCollection) { if (!string.IsNullOrWhiteSpace(cookie.Value)) { cookiesCollection[cookie.Name] = cookie; } } } if (antiScrappingCookie != null) { cookiesCollection[antiScrappingCookie.Name] = antiScrappingCookie; } return(cookiesCollection); }