public override async Task ProcessDataAsync() { var scrapedSaleAnnounces = new List <TransportSaleAnnounce>(); while (ItemsCountForStep > scrapedSaleAnnounces.Count) { try { var urlForScrapping = UrlForScrapping + $"&start={ScrappingPage}"; var htmlDocument = await DataLoader.LoadHtmlDocumentAsync(urlForScrapping); var previewOffers = StructureExtractor.GetPreviewOfferStructure(htmlDocument, ScrappingPage).ToList(); for (int i = 0; i < previewOffers.Count; i++) { tryAgain: try { var sourceLink = SourceLinkParser.GetLink(previewOffers[i], BaseUrl); var detailedHtmlDocument = await DataLoader.LoadHtmlDocumentAsync(sourceLink); var detailedOfferNode = StructureExtractor.GetDetailedOfferStructure(detailedHtmlDocument); var(brandId, modelId) = BrandModelParser.ParseForDetailed(detailedOfferNode); var saleAnnounce = new TransportSaleAnnounce() { SourceLink = sourceLink, TransmissionTypeId = TransmissionTypeParser.ParseForDetailed(detailedOfferNode), AdNumber = OfferNumberParser.ParseForDetailed(detailedOfferNode), BodyTypeId = BodyTypeParser.ParseForDetailed(detailedOfferNode), CityId = RegionParser.ParseForDetailed(detailedOfferNode).Id, Description = DescriptionParser.ParseForDetailed(detailedOfferNode), DriveUnitId = DriveUnitParser.ParseForDetailed(detailedOfferNode), EngineVolumetric = EngineVolumetricParser.ParseForDetailed(detailedOfferNode), FuelTypeId = FuelTypeParser.ParseForDetailed(detailedOfferNode), PreviewImageLink = ImageLinkParser.ParseForPreview(previewOffers[i]), Mileage = MileageParser.ParseForDetailed(detailedOfferNode), BrandId = brandId, VehicleTypeId = VehicleTypeParser.ParseForDetailed(detailedOfferNode), ModelId = modelId, PriceInDollars = PriceParser.ParseForDetailed(detailedOfferNode), UpdateOfferTime = PublishDateParser.ParseForDetailed(detailedOfferNode), Year = YearParser.ParseForDetailed(detailedOfferNode), CreatedAt = DateTime.Now, SourceProviderId = (int)SourceProviderEnum.RST }; var carConditionIds = CarConditionParser.ParseForPreview(previewOffers[i]); foreach (var carConditionId in carConditionIds) { saleAnnounce.TransportConditions.Add(new TransportConditionInSaleAnnounce() { TransportConditionId = carConditionId }); } scrapedSaleAnnounces.Add(saleAnnounce); UnitOfWork.TransportSaleAnnouncesRepository.Add(saleAnnounce); await UnitOfWork.SaveChangesAsync(); ScrappingPage++; } catch (WebException) { goto tryAgain; } } } catch (Exception ex) { _logger.LogError($"{ex.Message}\n{ex.StackTrace}"); } } //await UnitOfWork.TransportSaleAnnouncesRepository.InsertRangeSaleAnnounces(scrapedSaleAnnounces); }