示例#1
0
        public async Task ImportAsync()
        {
            var httpClient = new HttpClient {
                Timeout = ScrappingConfiguration.HttpClientTimeout
            };

            OfficialStoreInfo[] storesToImport = null;
            try
            {
                storesToImport =
                    await httpClient.GetFromJsonAsync <OfficialStoreInfo[]>("https://www.tuenvio.cu/stores.json");
            }
            catch (Exception e)
            {
                Log.Error(e, "Error requesting stores.json");

                // TODO: Improve this.
                CookiesHelper.InvalidateAntiScrappingCookie();
            }

            // TODO: Report the status as error.
            if (storesToImport != null)
            {
                var storesUrl = storesToImport.Select(store => new Uri(store.Url));
                foreach (var url in storesUrl)
                {
                    await this.AddAsync(url);
                }
            }
        }
示例#2
0
        private async Task <Store> GetDirectAsync(string url)
        {
            Log.Information("Scrapping Store from {Url}", url);

            OfficialStoreInfo[] storesToImport = null;
            try
            {
                storesToImport = await this.httpClient.GetFromJsonAsync <OfficialStoreInfo[]>("https://www.tuenvio.cu/stores.json");
            }
            catch (Exception e)
            {
                // TODO: Improve this?
                CookiesHelper.InvalidateAntiScrappingCookie();

                Log.Error(e, "Error requesting stores.json");
            }

            var    requestUri = url;
            string content    = null;

            try
            {
                content = await this.httpClient.GetStringAsync(requestUri);
            }
            catch (Exception e)
            {
                Log.Error(e, "Error requesting Store from '{url}'", url);
            }

            var storeToImport    = storesToImport?.FirstOrDefault(s => $"{s.Url.Trim()}/Products?depPid=0" == url.Trim());
            var storeName        = storeToImport?.Name;
            var isAvailable      = false;
            var categoriesCount  = 0;
            var departmentsCount = 0;

            if (!string.IsNullOrWhiteSpace(content))
            {
                var document = await this.browsingContext.OpenAsync(req => req.Content(content));

                if (string.IsNullOrWhiteSpace(storeName))
                {
                    var footerElement = document.QuerySelector <IElement>("#footer > div.container > div > div > p");
                    var uriParts      = url.Split('/');
                    if (uriParts.Length > 3)
                    {
                        storeName = url.Split('/')[3];
                    }

                    if (footerElement != null)
                    {
                        var footerElementTextParts = footerElement.TextContent.Split('•');
                        if (footerElementTextParts.Length > 0)
                        {
                            storeName = footerElementTextParts[^ 1].Trim();