private void createPlanForVehicleType(ITitleLinkPair vehicleType)
{
// web request - producers
VehicleTypeDocumentRequestor dr = new VehicleTypeDocumentRequestor(new WebRequestor(), vehicleType.Link);
VehicleTypeParser parser = new VehicleTypeParser(dr);
ITitleLinkPair[] producers = parser.Parse();
vehicleTypeProducers.Add(vehicleType.Title, producers);
}
public override async Task ProcessDataAsync()
{
var scrapedSaleAnnounces = new List <TransportSaleAnnounce>();
while (ItemsCountForStep > scrapedSaleAnnounces.Count)
{
try
{
var urlForScrapping = UrlForScrapping + $"&start={ScrappingPage}";
var htmlDocument = await DataLoader.LoadHtmlDocumentAsync(urlForScrapping);
var previewOffers = StructureExtractor.GetPreviewOfferStructure(htmlDocument, ScrappingPage).ToList();
for (int i = 0; i < previewOffers.Count; i++)
{
tryAgain:
try
{
var sourceLink = SourceLinkParser.GetLink(previewOffers[i], BaseUrl);
var detailedHtmlDocument = await DataLoader.LoadHtmlDocumentAsync(sourceLink);
var detailedOfferNode = StructureExtractor.GetDetailedOfferStructure(detailedHtmlDocument);
var(brandId, modelId) = BrandModelParser.ParseForDetailed(detailedOfferNode);
var saleAnnounce = new TransportSaleAnnounce()
{
SourceLink = sourceLink,
TransmissionTypeId = TransmissionTypeParser.ParseForDetailed(detailedOfferNode),
AdNumber = OfferNumberParser.ParseForDetailed(detailedOfferNode),
BodyTypeId = BodyTypeParser.ParseForDetailed(detailedOfferNode),
CityId = RegionParser.ParseForDetailed(detailedOfferNode).Id,
Description = DescriptionParser.ParseForDetailed(detailedOfferNode),
DriveUnitId = DriveUnitParser.ParseForDetailed(detailedOfferNode),
EngineVolumetric = EngineVolumetricParser.ParseForDetailed(detailedOfferNode),
FuelTypeId = FuelTypeParser.ParseForDetailed(detailedOfferNode),
PreviewImageLink = ImageLinkParser.ParseForPreview(previewOffers[i]),
Mileage = MileageParser.ParseForDetailed(detailedOfferNode),
BrandId = brandId,
VehicleTypeId = VehicleTypeParser.ParseForDetailed(detailedOfferNode),
ModelId = modelId,
PriceInDollars = PriceParser.ParseForDetailed(detailedOfferNode),
UpdateOfferTime = PublishDateParser.ParseForDetailed(detailedOfferNode),
Year = YearParser.ParseForDetailed(detailedOfferNode),
CreatedAt = DateTime.Now,
SourceProviderId = (int)SourceProviderEnum.RST
};
var carConditionIds = CarConditionParser.ParseForPreview(previewOffers[i]);
foreach (var carConditionId in carConditionIds)
{
saleAnnounce.TransportConditions.Add(new TransportConditionInSaleAnnounce()
{
TransportConditionId = carConditionId
});
}
scrapedSaleAnnounces.Add(saleAnnounce);
UnitOfWork.TransportSaleAnnouncesRepository.Add(saleAnnounce);
await UnitOfWork.SaveChangesAsync();
ScrappingPage++;
}
catch (WebException)
{
goto tryAgain;
}
}
}
catch (Exception ex)
{
_logger.LogError($"{ex.Message}\n{ex.StackTrace}");
}
}
//await UnitOfWork.TransportSaleAnnouncesRepository.InsertRangeSaleAnnounces(scrapedSaleAnnounces);
}
