public async Task ImportAsync()
{
var httpClient = new HttpClient {
Timeout = ScrappingConfiguration.HttpClientTimeout
};
OfficialStoreInfo[] storesToImport = null;
try
{
storesToImport =
await httpClient.GetFromJsonAsync <OfficialStoreInfo[]>("https://www.tuenvio.cu/stores.json");
}
catch (Exception e)
{
Log.Error(e, "Error requesting stores.json");
// TODO: Improve this.
CookiesHelper.InvalidateAntiScrappingCookie();
}
// TODO: Report the status as error.
if (storesToImport != null)
{
var storesUrl = storesToImport.Select(store => new Uri(store.Url));
foreach (var url in storesUrl)
{
await this.AddAsync(url);
}
}
}
private async Task <Store> GetDirectAsync(string url)
{
Log.Information("Scrapping Store from {Url}", url);
OfficialStoreInfo[] storesToImport = null;
try
{
storesToImport = await this.httpClient.GetFromJsonAsync <OfficialStoreInfo[]>("https://www.tuenvio.cu/stores.json");
}
catch (Exception e)
{
// TODO: Improve this?
CookiesHelper.InvalidateAntiScrappingCookie();
Log.Error(e, "Error requesting stores.json");
}
var requestUri = url;
string content = null;
try
{
content = await this.httpClient.GetStringAsync(requestUri);
}
catch (Exception e)
{
Log.Error(e, "Error requesting Store from '{url}'", url);
}
var storeToImport = storesToImport?.FirstOrDefault(s => $"{s.Url.Trim()}/Products?depPid=0" == url.Trim());
var storeName = storeToImport?.Name;
var isAvailable = false;
var categoriesCount = 0;
var departmentsCount = 0;
if (!string.IsNullOrWhiteSpace(content))
{
var document = await this.browsingContext.OpenAsync(req => req.Content(content));
if (string.IsNullOrWhiteSpace(storeName))
{
var footerElement = document.QuerySelector <IElement>("#footer > div.container > div > div > p");
var uriParts = url.Split('/');
if (uriParts.Length > 3)
{
storeName = url.Split('/')[3];
}
if (footerElement != null)
{
var footerElementTextParts = footerElement.TextContent.Split('•');
if (footerElementTextParts.Length > 0)
{
storeName = footerElementTextParts[^ 1].Trim();