public async Task Start()
{
const int max = 25;
for (int i = 0; i < max; i++)
{
Console.WriteLine("loading pap fr page " + i + 1);
var baseAddress = "https://www.pap.fr/annonce/propriete-en-vente-ile-de-france-g471-" + i;
var config = Configuration.Default.WithDefaultLoader();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(baseAddress);
var list = document.QuerySelectorAll(".main-content .search-results-list .search-list-item");
foreach (var element in list)
{
var model = new GenericAnnouncementModel("pap.fr");
var imgSrc = element.QuerySelector(" div.col-left > a > img")?.Attributes["src"]?.Value;
var url = element.QuerySelector(" div.col-left > a")?.Attributes["href"]?.Value;
var photoCount = element.QuerySelector(" div.col-left > .item-photo-count")?.Text();
if (!url.StartsWith("https://"))
{
url = "https://www.pap.fr" + url;
}
var title = element.QuerySelector("div.col-right > a.item-title > span.h1")?.TextContent;
var itemTags = element.QuerySelectorAll("div.col-right > a.item-title > ul > li")?.Select(li => li?.TextContent).ToArray();
var itemPrice = element.QuerySelector(".col-right .item-price")?.TextContent;
var priceDescription = element.QuerySelector(".col-right .mensualite-prix")?.TextContent;
var itemDescription = element.QuerySelector(".col-right > p.item-description")?.TextContent;
var itemTransport = element.QuerySelector(".col-right > .item-transports")?.TextContent;
var id = url?.Split('/')?.Last();
//id = id?.Replace("r", "");
if (string.IsNullOrEmpty(id))
{
id = string.Empty;
}
if (id.Contains("?"))
{
id = id.Split('?').First();
}
if (id.Contains("-"))
{
id = id.Split('-').Last();
}
model.ID = id;
model.Images = new List <string> {
imgSrc
};
model.LinkUrl = url;
model.Title = title;
model.Price = new List <string> {
itemPrice
};
model.Description = itemDescription;
model.Extra = new { photoCount, itemTags, priceDescription, itemTransport };
AnnouncementParsed?.Invoke(model);
}
}
}
public async Task Start()
{
string templateUrl =
"https://www.seloger.com/list.htm?tri=initial&idtypebien=2,1&pxMax=1000000&div=2238&idtt=2,5&naturebien=1,2,4";
templateUrl =
"https://www.seloger.com/list.htm?tri=initial&idtypebien=2,1&pxMax=100000000&div=2238&idtt=2,5&naturebien=1,2,4";
templateUrl =
"https://www.seloger.com/list.htm?tri=initial&idtypebien=2,1&idtt=2,5&naturebien=1,2,4&idPays=250";
int i = 1;
while (true)
{
Logger.WriteLine("\nloading seloger page: " + i);
var url = templateUrl + "&LISTING-LISTpg=" + i;
var config = Configuration.Default.WithDefaultLoader();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(url);
var html = document.ToHtml();
if (html.Contains("Aucun résultat pour l'instant"))
{
Logger.WriteLine("seloger completed");
return;
}
if (html.Contains("Oops, une erreur technique est survenue. Merci de ressayer ultérieurement."))
{
//error cixib
Logger.WriteLine("Seloger Server down waiting for 10 sec");
Thread.Sleep(TimeSpan.FromSeconds(10));
i++;
continue;
}
var listResult = document.QuerySelector("section.liste_resultat");
if (listResult == null)
{
i++;
continue;
}
var items = listResult.QuerySelectorAll(".c-pa-list")?.ToArray();
if (items == null)
{
i++;
continue;
}
foreach (var item in items)
{
var model = ParseAnnouncement(item);
AnnouncementParsed?.Invoke(model);
}
i++;
}
}
public async Task Start()
{
var config = Configuration.Default.WithDefaultLoader();
const string baseAddress = "https://www.superimmo.com/achat/aquitaine/";
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(baseAddress);
var aSelectors = "ul.pagination li.large-sm a";
var apartsSelector = "article.appart_view";
var cells = document.QuerySelectorAll(aSelectors);
int pageCount;
int.TryParse(cells?.Last()?.TextContent, out pageCount);
var aparts = document.QuerySelectorAll(apartsSelector).ToArray();
foreach (IElement apart in aparts)
{
AnnouncementParsed?.Invoke(ParseAnnouncement(apart));
}
for (int i = 2; i <= pageCount; i++)
{
Logger.WriteLine("loading superimmo on page " + i + "/" + pageCount);
var url = baseAddress + "p/" + i;
context = BrowsingContext.New(config);
document = await context.OpenAsync(url);
aparts = document.QuerySelectorAll(apartsSelector).ToArray();
foreach (IElement apart in aparts)
{
AnnouncementParsed?.Invoke(ParseAnnouncement(apart));
}
}
Logger.WriteLine("superimmo completed");
//var titles = cells.Select(m => m.TextContent);
}
public Task Start()
{
var regionsLength = GetRegions().Count();
for (int i = 1; i <= regionsLength; i++)
{
var all = ParseAll(i);
var dict = new List <string>();
foreach (GenericAnnouncementModel modelEx in all)
{
if (dict.Contains(modelEx.ID))
{
break;
}
dict.Add(modelEx.ID);
AnnouncementParsed?.Invoke(modelEx);
}
}
Logger.WriteLine("Leboncoin completed");
return(Task.CompletedTask);
}
