private async Task <bool> _downloadItemAsync(AdDtoModel item, ScraperHomeLessStateModel state)
{
var result = true;
var id = item.Id;
try
{
var adDetails = _getAdDetailsFromService(item);
var coordinates = _getCoordinatesFromServiceAsync(item, state);
var phones = _getPhonesFromService(item);
var details = _getDetailsFromService(item);
var detailsItemDto = new DetailsItemDtoModel
{
Coordinates = await coordinates,
Phones = await phones,
Details = await details,
AdDetails = await adDetails,
RowDataFromPage = item,
};
await _saveItemDetailsAsync(id, detailsItemDto, state);
}
catch (Exception exception)
{
_log($"Error-f1. {exception.Message} / {exception.StackTrace}");
result = false;
}
return(result);
}
private void _scrapeFromPage_Rent(List <AdDtoModel> result, EnumTypeItems typeItem)
{
var idBlock = "mainresults";
switch (typeItem)
{
case EnumTypeItems.Rent:
idBlock = "mainresults";
break;
case EnumTypeItems.RentTivuch:
idBlock = "relatedresults";
break;
}
var data = _selenoidState.WindowMain.FindElementById(idBlock);//board mainresults/relatedresults rent/RentTivuch
var rows = data.FindElements(By.TagName("tr"));
foreach (var row in rows)
{
var listTds = row.FindElements(By.TagName("td"));
if (listTds.Count > 0 && listTds.FirstOrDefault().GetAttribute("class") == "selectionarea")
{
var id = row.GetAttribute("id");
var city = listTds.Skip(5).Take(1).FirstOrDefault().Text; //6
var region = listTds.Skip(4).Take(1).FirstOrDefault().Text; //5
var price = listTds.Skip(7).Take(1).FirstOrDefault().Text; //8
var dateUpdated = listTds.Skip(9).Take(1).FirstOrDefault().Text; //10
var item = new AdDtoModel()
{
TypeItem = typeItem,
Id = id.Replace("ad_", ""),
City = city,
Region = region,
Price = price,
DateUpdated = dateUpdated,
};
result.Add(item);
}
}
}
private async Task <string> _getPhonesFromService(AdDtoModel item)
{
var result = "";
var id = item.Id;
var board = item.TypeItem.ToString();
var url = $"https://www.homeless.co.il/webservices/icardos.asmx/IncrementClickesAndGetPhoneNumber";
var request1 = url
.WithTimeout(60)
.WithHeaders(new
{
User_Agent = "wow hackers, I need your money",
})
.PostJsonAsync(new { boardType = board, AdID = id })
.ReceiveJson <PostPhonesDtoModel>();
var phones = await request1;
result = phones.d;
return(result);
}
private async Task <AdDetailsDtoModel> _getAdDetailsFromService(AdDtoModel item)
{
AdDetailsDtoModel result = null;
PostAdDetailsDtoModel postData = null;
var id = item.Id;
var request = "";
var board = item.TypeItem.ToString();
var needDo = false;
var url = $"https://www.homeless.co.il/webservices/icardos.asmx/Cpart_GetAdDetails?boardid={board}&itemid={id}";
do
{
needDo = false;
try
{
request = await url
.WithHeaders(new
{
User_Agent = "wow hackers, I need your money",
Accept = "*/*",
Content_Type = "application/json; charset=utf-8",
})
.PostJsonAsync(new { boardid = board, itemid = id })
.ReceiveString();
postData = JsonConvert.DeserializeObject <PostAdDetailsDtoModel>(request);
result = postData.d;
}
catch (Exception exception)
{
_log($"Error d4. Wait 1 sec. {exception.Message} ");
needDo = true;
Thread.Sleep(1000);
}
} while (needDo);
return(result);
}
private async Task <List <PostDetailsDDtoModel> > _getDetailsFromService(AdDtoModel item)
{
PostDetailsDtoModel details = null;
List <PostDetailsDDtoModel> result = null;
string request = "";
var id = item.Id;
var board = item.TypeItem.ToString();
var needDo = false;
var url = $"https://www.homeless.co.il/webservices/icardos.asmx/GetItemDetails";
do
{
needDo = false;
try
{
request = await url
.WithTimeout(60)
.WithHeaders(new
{
User_Agent = "wow hackers, I need your money",
})
.PostJsonAsync(new { boardid = board, itemid = id })
.ReceiveString();
}
catch (Exception exception)
{
_log($"Error d3. {exception.Message} ");
needDo = true;
Thread.Sleep(1000);
}
} while (needDo);
details = ParseDetailsFromService(request);
result = details.D;
return(result);
}
private AdDtoModel ParseAdsFromPage_ParseAd(HtmlNode row, EnumTypeItems typeItem)
{
AdDtoModel result = null;
var listTds = row.CssSelect("td").ToList();
var attr = listTds?.FirstOrDefault()?.Attributes["class"]?.Value;
var positionDate = ParseAdsFromPage_ParseAd_GetPositionDate(typeItem);
if (listTds.Count > 0 && attr == "selectionarea")
{
string id = "", city = "", region = "", price = "", dateUpdated = "";
try
{
id = row.Attributes["id"].Value.Replace("ad_", "");
city = listTds.Skip(5).Take(1).FirstOrDefault().ChildNodes.FirstOrDefault().InnerHtml; //6
region = listTds.Skip(4).Take(1).FirstOrDefault().InnerHtml; //5
price = listTds.Skip(7).Take(1).FirstOrDefault().InnerHtml; //8
dateUpdated = listTds.Skip(positionDate - 1).Take(1).FirstOrDefault()?.ChildNodes?.FirstOrDefault()?.InnerHtml; //10, revuch=11
}catch (Exception exception)
{
_log($"Error x4. {exception.Message} / {exception.StackTrace}");
}
result = new AdDtoModel()
{
TypeItem = typeItem,
Id = id,
City = city,
Region = region,
Price = price,
DateUpdated = dateUpdated,
};
}
return(result);
}
private async Task <List <ScraperModels.Models.CoordinateDtoModel> > _getCoordinatesFromServiceAsync(AdDtoModel item, ScraperHomeLessStateModel state)
{
List <ScraperModels.Models.CoordinateDtoModel> coordinates = await GetCoordinatesFrom_OpenStreetMap_WebClientProxyAsync($"{item.Region} {item.City}", state);
if (coordinates is null)
{
//coordinates = await _getCoordinatesFromService_ProxyAsync(item);
coordinates = await GetCoordinatesFrom_OpenStreetMap_WebClientProxyAsync($"{item.Region} {item.City}", state);
}
_log($"Wait 10 secs", state);
Thread.Sleep(1000 * 10);
return(coordinates);
}
