private async Task <bool> _downloadItemAsync(AdDtoModel item, ScraperHomeLessStateModel state)
{
var result = true;
var id = item.Id;
try
{
var adDetails = _getAdDetailsFromService(item);
var coordinates = _getCoordinatesFromServiceAsync(item, state);
var phones = _getPhonesFromService(item);
var details = _getDetailsFromService(item);
var detailsItemDto = new DetailsItemDtoModel
{
Coordinates = await coordinates,
Phones = await phones,
Details = await details,
AdDetails = await adDetails,
RowDataFromPage = item,
};
await _saveItemDetailsAsync(id, detailsItemDto, state);
}
catch (Exception exception)
{
_log($"Error-f1. {exception.Message} / {exception.StackTrace}");
result = false;
}
return(result);
}
//private async Task<List<ScraperModels.Models.HomeLess.CoordinateDtoModel>> _getCoordinatesFromService_WebClientAsync(AdDtoModel item)
//{
// List<ScraperModels.Models.HomeLess.CoordinateDtoModel> coordinates = null;
// var url = $"https://nominatim.openstreetmap.org/search?format=json&q={item.Region} {item.City}";
// var response = "";
// var needRepeat = false;
// var indexRepeat=0;
// do
// {
// needRepeat = false;
// try
// {
// response = await url
// .WithHeaders(new {
// User_Agent = "PostmanISC/Israel",
// })
// .GetStringAsync();
// coordinates = JsonConvert.DeserializeObject<List<ScraperModels.Models.HomeLess.CoordinateDtoModel>>(response);
// }
// catch (Exception exception)
// {
// indexRepeat++;
// _log($"Error g1. Coordinates wo proxy. Wait 120 sec. indexRepeat {indexRepeat} {exception.Message} / URL: {url}");
// if (indexRepeat<10) needRepeat = true;
// Thread.Sleep(1000 * 120);
// }
// } while (needRepeat);
// return coordinates;
//}
//private async Task<List<ScraperModels.Models.HomeLess.CoordinateDtoModel>> _getCoordinatesFromService_ProxyAsync(AdDtoModel item)
//{
// List<ScraperModels.Models.HomeLess.CoordinateDtoModel> coordinates = null;
// var client = new WebClient();
// var indexRepeat = 0;
// ICredentials credentials = new NetworkCredential("lum-customer-hl_89055c51-zone-static", "y7ic12hyfl9b");
// client.Proxy = new WebProxy(new Uri("http://zproxy.lum-superproxy.io:22225"), true, null, credentials);
// client.Headers.Add("User-Agent", "Mozilla/6.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20200101 Firefox/89.0");
// var url = $"https://nominatim.openstreetmap.org/search?format=json&q={item.Region} {item.City}";
// var response = "";
// var needRepeat = false;
// do
// {
// needRepeat = false;
// try
// {
// response = await client.DownloadStringTaskAsync(new Uri(url));
// coordinates = JsonConvert.DeserializeObject<List<ScraperModels.Models.HomeLess.CoordinateDtoModel>>(response);
// }
// catch (Exception exception)
// {
// _log($"Error g2. Coordinates. Wait 120 sec. indexRepeat {indexRepeat} {exception.Message} / URL: {url}");
// indexRepeat++;
// if (indexRepeat < 10) needRepeat = true;
// Thread.Sleep(1000 * 120);
// }
// } while (needRepeat);
// return coordinates;
//}
private async Task _saveItemDetailsAsync(string id, DetailsItemDtoModel response, ScraperHomeLessStateModel state)
{
var filename = $"{state.ItemsPath}/{id}.json";
await File.WriteAllTextAsync(filename, JsonConvert.SerializeObject(response, Newtonsoft.Json.Formatting.Indented));
_log($"Save json file {filename}");
}
private DetailsItemDtoModel _loadDetailsItem(FileInfo file)
{
DetailsItemDtoModel result = null;
var id = Path.GetFileNameWithoutExtension(file.Name);
var filename = $"{Path.GetDirectoryName(file.FullName)}/{id}.json";
if (File.Exists(filename))
{
result = JsonConvert.DeserializeObject <DetailsItemDtoModel>(File.ReadAllText(filename));
}
return(result);
}
