public bool UpdateData(DataDomainModel data) { var result = true; Truncate(); var items = (List <AdItemWinWinDomainModel>)data.Data; var index = 0; foreach (var item in items) { var itemDb = new AdItemWinWinDbModel().FromDomain(item); _context.DataWinWin.Add(itemDb); index++; if (index % 100 == 0) { Console.WriteLine($"index={index}"); _context.SaveChanges(); } } _context.SaveChanges(); UpdateGeometry(); return(result); }
public bool UpdateData(DataDomainModel data) { var result = true; Truncate(); var items = (List <AdItemHomeLessDomainModel>)data.Data; Console.WriteLine($"Need update items: {items.Count()}"); var index = 0; foreach (var item in items) { if (item.IsValidForDbModel) { var itemDb = new AdItemHomeLessDbModel().FromDomain(item); _context.DataHomeLess.Add(itemDb); index++; if (index % 300 == 0) { Console.WriteLine($"index={index}"); _context.SaveChanges(); } } } _context.SaveChanges(); UpdateGeometry(); return(result); }
public DataDomainModel SaveDomainModel(DataDomainModel model) { var state = _state; //var model = GetDomainModelAsync().Result; var filename = $"{state.RootPath}/domain-model.json"; File.WriteAllText(filename, JsonConvert.SerializeObject(model, Formatting.Indented)); return(model); }
/// <summary> /// Deletes the domain. /// </summary> /// <param name="id">The domain's id.</param> /// <returns>Refresh the page.</returns> public async Task <IActionResult> OnPostDeleteAsync(int id) { DomainData domainData = new DomainData(_db); DataDomainModel newDomainModel = new DataDomainModel { Domain = DisplayDomain.Domain, Id = DisplayDomain.Id, }; await domainData.DeleteDomain(newDomainModel); return(RedirectToPage("./domain")); }
public DataDomainModel GetDataFromScraper(RequestDataScrape request) { var result = new DataDomainModel(); switch (request.ScrapeId) { case EnumScrapers.Airdna: var scraper = new AirdnaScraper(); result = scraper.Scrape(request.CityId); break; } return(result); }
public override async Task <DataDomainModel> GetDomainModelAsync() { var state = (ScraperHomeLessStateModel)_state; SetWorkPhaseBase($"DomainModel", state); var listDomainItems = ScrapePhase4Async(state); var result = new DataDomainModel() { Scraper = state.TypeScraper, Data = await listDomainItems, }; LogDone(state); return(result); }
public async override Task <DataDomainModel> GetDomainModelAsync() { var state = (ScraperOnmapStateModel)_state; SetWorkPhaseBase($"DomainModel", state); var domainModel = ScrapePhase3Async(state); var result = new DataDomainModel() { Scraper = EnumScrapers.Onmap, Data = await domainModel, }; LogDone(state); return(result); }
public override async Task <DataDomainModel> GetDomainModelAsync() { var state = (ScraperYad2StateModel)_state; SetWorkPhaseBase($"DomainModel", state); var domainModel = _scrapePhase7_GenerateDomainModelAsync(state); DataDomainModel result = new DataDomainModel() { Scraper = EnumScrapers.Yad2, Data = await domainModel, }; LogDone(state); return(result); }
public async override Task <DataDomainModel> GetDomainModelAsync() { var state = _state; SetWorkPhaseBase($"DomainModel", state); var listDomainItems = await ScrapePhase_GetDomainModelAsync(state); var result = new DataDomainModel() { Scraper = state.TypeScraper, Data = listDomainItems, }; _logBase($"DomainModel has {listDomainItems.Count()} AdItems", state); LogDone(state); return(result); }
/// <summary> /// Creates a new domain. /// </summary> /// <returns>Refresh the page.</returns> public async Task <IActionResult> OnPost() { if (!ModelState.IsValid) { return(Page()); } DomainData domainData = new DomainData(_db); DataDomainModel newDomainModel = new DataDomainModel { Domain = DisplayDomain.Domain, Id = DisplayDomain.Id, }; await domainData.InsertDomain(newDomainModel); return(RedirectToPage("./domain")); }
public MemoryStream CreateExcel(DataDomainModel data) { MemoryStream result = null; var itemsDomainModel = (List <AdItemYad2DomainModel>)data.Data; var items = new List <AdItemYad2ExcelModel>(); foreach (var item in itemsDomainModel) { items.Add(new AdItemYad2ExcelModel().FromDomain(item)); } var amountDataCols = 0; var hasAmountImages = 1; _log($"Amount input items: {items.Count}"); using (ExcelPackage eP = new ExcelPackage()) { eP.Workbook.Properties.Author = "Scrap"; eP.Workbook.Properties.Title = "Scrap Data"; eP.Workbook.Properties.Company = "Yad2"; var sheet = eP.Workbook.Worksheets.Add($"Yad2"); var row = 1; var col = 1; // head line sheet.Cells[row, col++].Value = "ItemId"; sheet.Cells[row, col++].Value = "Date Create"; sheet.Cells[row, col++].Value = "Date Update"; sheet.Cells[row, col++].Value = "Latitude"; sheet.Cells[row, col++].Value = "Longitude"; sheet.Cells[row, col++].Value = "City"; sheet.Cells[row, col++].Value = "HouseNumber"; sheet.Cells[row, col++].Value = "Neighborhood"; sheet.Cells[row, col++].Value = "StreetName"; sheet.Cells[row, col++].Value = "AriaBase"; sheet.Cells[row, col++].Value = "Balconies"; sheet.Cells[row, col++].Value = "Pets"; sheet.Cells[row, col++].Value = "Elevators"; sheet.Cells[row, col++].Value = "FloorOn"; sheet.Cells[row, col++].Value = "FloorOf"; sheet.Cells[row, col++].Value = "Rooms"; sheet.Cells[row, col++].Value = "Parking"; //sheet.Cells[row, col++].Value = "ContactEmail"; sheet.Cells[row, col++].Value = "ContactName"; sheet.Cells[row, col++].Value = "ContactPhone"; sheet.Cells[row, col++].Value = "Price"; var descriptionDataCol = col; sheet.Cells[row, col++].Value = "Description"; sheet.Cells[row, col++].Value = "PropertyType"; sheet.Cells[row, col++].Value = "AirConditioner"; amountDataCols = col; sheet.Cells[row, col++].Value = "Link"; //sheet.Cells[row, 1, row, 23].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center; sheet.Cells[1, 1, 1, amountDataCols - 1].AutoFilter = true; row++; col = 1; foreach (var item in items) { col = 1; sheet.Cells[row, col++].Value = item.ItemId; _addCellDate(sheet.Cells[row, col], item.DateCreate); col++; _addCellDate(sheet.Cells[row, col], item.DateUpdate); col++; sheet.Cells[row, col++].Value = item.Latitude; sheet.Cells[row, col++].Value = item.Longitude; sheet.Cells[row, col++].Value = item.HeCity; _addCellInt(sheet.Cells[row, col], item.HeHouseNumber); col++; sheet.Cells[row, col++].Value = item.HeNeighborhood; sheet.Cells[row, col++].Value = item.HeStreetName; _addCellInt(sheet.Cells[row, col], item.AriaBase); col++; _addCellInt(sheet.Cells[row, col], item.Balconies); col++; sheet.Cells[row, col++].Value = item.Pets; sheet.Cells[row, col++].Value = item.Elevators; _addCellInt(sheet.Cells[row, col], item.FloorOn); col++; _addCellInt(sheet.Cells[row, col], item.FloorOf); col++; _addCellInt(sheet.Cells[row, col], item.Rooms); col++; sheet.Cells[row, col++].Value = item.Parking; //sheet.Cells[row, col++].Value = item.ContactEmail; sheet.Cells[row, col++].Value = item.ContactName; sheet.Cells[row, col++].Value = item.ContactPhone; sheet.Cells[row, col++].Value = item.Price; sheet.Cells[row, col++].Value = item.Description; sheet.Cells[row, col++].Value = item.PropertyType; sheet.Cells[row, col++].Value = item.AirConditioner; // link var url = $"https://www.yad2.co.il/item/{item.ItemId}"; sheet.Cells[row, col].Value = url; sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center; sheet.Cells[row, col].Hyperlink = new Uri(url); col++; if (item?.Images != null) { var i = 1; foreach (var image in item.Images) { _addCellLinks(sheet.Cells[row, col], image, i++); col++; } if (item.Images.Count > hasAmountImages) { hasAmountImages = item.Images.Count; } } row++; } col--; using (var cells = sheet.Cells[sheet.Cells[1, 1, 1 + items.Count, amountDataCols + hasAmountImages].Address]) { cells.Style.Border.Top.Style = ExcelBorderStyle.Thin; cells.Style.Border.Bottom.Style = ExcelBorderStyle.Thin; cells.Style.Border.Right.Style = ExcelBorderStyle.Thin; cells.Style.Border.Left.Style = ExcelBorderStyle.Thin; cells.AutoFitColumns(); } sheet.Column(descriptionDataCol).Width = 50; var startPositionOnFileLinks = amountDataCols + 1; var endPositionOnFileLinks = startPositionOnFileLinks + hasAmountImages; foreach (var i in Enumerable.Range(startPositionOnFileLinks, hasAmountImages)) { sheet.Cells[1, i].Value = $"Images {i - startPositionOnFileLinks + 1}"; } foreach (var i in Enumerable.Range(3, row)) { sheet.Row(i).Height = 15; } result = new MemoryStream(eP.GetAsByteArray()); } return(result); }
public MemoryStream CreateExcel(DataDomainModel data) { _log($"Start create excel data"); MemoryStream result = null; var itemsDomainModel = (List <AdItemWinWinDomainModel>)data.Data; var items = new List <AdItemWinWinExcelModel>(); foreach (var item in itemsDomainModel) { items.Add(new AdItemWinWinExcelModel().FromDomain(item)); } var amountDataCols = 0; var hasAmountImages = 1; _log($"Amount input items: {items.Count}"); using (ExcelPackage eP = new ExcelPackage()) { eP.Workbook.Properties.Author = "Scrap"; eP.Workbook.Properties.Title = "Scrap Data"; eP.Workbook.Properties.Company = "WinWin"; var sheet = eP.Workbook.Worksheets.Add($"WinWin"); var row = 1; var col = 1; // head line sheet.Cells[row, col++].Value = "ItemId"; sheet.Cells[row, col++].Value = "Date Update"; sheet.Cells[row, col++].Value = "Longitude"; sheet.Cells[row, col++].Value = "Latitude"; sheet.Cells[row, col++].Value = "City"; sheet.Cells[row, col++].Value = "Area"; sheet.Cells[row, col++].Value = "StreetAddress"; sheet.Cells[row, col++].Value = "Rooms"; sheet.Cells[row, col++].Value = "Floor"; sheet.Cells[row, col++].Value = "Status"; sheet.Cells[row, col++].Value = "DateEnter"; sheet.Cells[row, col++].Value = "Square"; sheet.Cells[row, col++].Value = "IsPartners"; sheet.Cells[row, col++].Value = "AmountPayment"; var descriptionDataCol = col; sheet.Cells[row, col++].Value = "Description"; sheet.Cells[row, col++].Value = "Price"; sheet.Cells[row, col++].Value = "IsAgent"; sheet.Cells[row, col++].Value = "ContactName"; sheet.Cells[row, col++].Value = "Phone1"; sheet.Cells[row, col++].Value = "Phone2"; amountDataCols = col; sheet.Cells[row, col++].Value = "Link"; //sheet.Cells[row, 1, row, col - 1].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center; sheet.Cells[1, 1, 1, amountDataCols - 1].AutoFilter = true; row++; col = 1; foreach (var item in items) { col = 1; sheet.Cells[row, col++].Value = item.ItemId; _addCellDate(sheet.Cells[row, col], item.DateUpdate); col++; sheet.Cells[row, col++].Value = item.Latitude; sheet.Cells[row, col++].Value = item.Longitude; sheet.Cells[row, col++].Value = item.City; _addCellInt(sheet.Cells[row, col], item.Area); col++; sheet.Cells[row, col++].Value = item.StreetAddress; _addCellInt(sheet.Cells[row, col], item.Rooms); col++; _addCellInt(sheet.Cells[row, col], item.Floor); col++; sheet.Cells[row, col++].Value = item.State; _addCellDate(sheet.Cells[row, col], item.DateEnter); col++; _addCellInt(sheet.Cells[row, col], item.Square); col++; sheet.Cells[row, col++].Value = item.IsPartners; sheet.Cells[row, col++].Value = item.AmountPayment; sheet.Cells[row, col++].Value = item.Description; _addCellInt(sheet.Cells[row, col], item.Price); col++; sheet.Cells[row, col++].Value = item.IsAgent; sheet.Cells[row, col++].Value = item.ContactName; sheet.Cells[row, col++].Value = item.Phone1; sheet.Cells[row, col++].Value = item.Phone2; // link var url = $"https://www.winwin.co.il/RealEstate/ForRent/Ads/RealEstateAds,{item.ItemId}.aspx"; sheet.Cells[row, col].Value = url; sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center; sheet.Cells[row, col].Hyperlink = new Uri(url); col++; var i = 1; foreach (var image in item.Images) { _addCellLinks(sheet.Cells[row, col], image.Full, i++); col++; } if (item.Images.Count > hasAmountImages) { hasAmountImages = item.Images.Count; } row++; } col--; using (var cells = sheet.Cells[sheet.Cells[1, 1, 1 + items.Count, amountDataCols + hasAmountImages].Address]) { cells.Style.Border.Top.Style = ExcelBorderStyle.Thin; cells.Style.Border.Bottom.Style = ExcelBorderStyle.Thin; cells.Style.Border.Right.Style = ExcelBorderStyle.Thin; cells.Style.Border.Left.Style = ExcelBorderStyle.Thin; cells.AutoFitColumns(); } sheet.Column(descriptionDataCol).Width = 50; var startPositionOnFileLinks = amountDataCols + 1; var endPositionOnFileLinks = startPositionOnFileLinks + hasAmountImages; foreach (var i in Enumerable.Range(startPositionOnFileLinks, hasAmountImages)) { sheet.Cells[1, i].Value = $"Images {i - startPositionOnFileLinks + 1}"; } foreach (var i in Enumerable.Range(3, row)) { sheet.Row(i).Height = 15; } result = new MemoryStream(eP.GetAsByteArray()); } _log("Excel data create done"); return(result); }
public MemoryStream CreateExcel(DataDomainModel data) { _log($"Start create excel data"); MemoryStream result = null; var itemsDomainModel = (List <AdItemAirdnaDomainModel>)data.Data; var items = new List <AdItemAirdnaExcelModel>(); foreach (var item in itemsDomainModel) { items.Add(new AdItemAirdnaExcelModel().FromDomain(item)); } _log($"Amount input items: {items.Count}"); using (ExcelPackage eP = new ExcelPackage()) { eP.Workbook.Properties.Author = "Scrap"; eP.Workbook.Properties.Title = "Scrap Data"; eP.Workbook.Properties.Company = "Airdna"; var sheet = eP.Workbook.Worksheets.Add($"Airdna"); var row = 1; var col = 1; // head line var descriptionDataCol = col; sheet.Cells[row, col++].Value = "Title (link to site)"; sheet.Cells[row, col++].Value = "Location"; sheet.Cells[row, col++].Value = "Latitude"; sheet.Cells[row, col++].Value = "Longitude"; sheet.Cells[row, col++].Value = "Adr"; sheet.Cells[row, col++].Value = "Rating"; sheet.Cells[row, col++].Value = "Bathrooms"; sheet.Cells[row, col++].Value = "Bedrooms"; sheet.Cells[row, col++].Value = "Accommodates"; sheet.Cells[row, col++].Value = "Revenue"; sheet.Cells[row, col++].Value = "Property Type"; sheet.Cells[row, col++].Value = "Occ"; sheet.Cells[row, col++].Value = "Reviews"; sheet.Cells[row, col++].Value = "RoomType"; sheet.Cells[row, col++].Value = "Link"; row++; col = 1; foreach (var item in items) { col = 1; sheet.Cells[row, col++].Value = item.Title; sheet.Cells[row, col++].Value = item.Location; sheet.Cells[row, col++].Value = item.Latitude; sheet.Cells[row, col++].Value = item.Longitude; sheet.Cells[row, col++].Value = item.Adr; sheet.Cells[row, col++].Value = item.Rating; sheet.Cells[row, col++].Value = item.Bathrooms; sheet.Cells[row, col++].Value = item.Bedrooms; sheet.Cells[row, col++].Value = item.Accommodates; sheet.Cells[row, col++].Value = item.Revenue; sheet.Cells[row, col++].Value = item.PropertyType; sheet.Cells[row, col++].Value = item.Occ; sheet.Cells[row, col++].Value = item.Reviews; sheet.Cells[row, col++].Value = item.RoomType; // link sheet.Cells[row, col].Value = item.LinkToProfile; sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center; sheet.Cells[row, col].Hyperlink = new Uri(item.LinkToProfile); col++; row++; } col--; sheet.Column(descriptionDataCol).Width = 50; result = new MemoryStream(eP.GetAsByteArray()); } _log("Excel data create done"); return(result); }
public MemoryStream CreateExcel(DataDomainModel data) { _log($"Start create excel data"); MemoryStream result = null; var itemsDomainModel = (List <AdItemHomeLessDomainModel>)data.Data; var items = new List <AdItemHomeLessExcelModel>(); foreach (var item in itemsDomainModel) { items.Add(new AdItemHomeLessExcelModel().FromDomain(item)); } var amountDataCols = 0; var hasAmountImages = 1; _log($"Amount input items: {items.Count}"); using (ExcelPackage eP = new ExcelPackage()) { eP.Workbook.Properties.Author = "Scrap"; eP.Workbook.Properties.Title = "Scrap Data"; eP.Workbook.Properties.Company = "HomeLess"; var sheet = eP.Workbook.Worksheets.Add($"HomeLess"); var row = 1; var col = 1; // head line sheet.Cells[row, col++].Value = "ItemId"; sheet.Cells[row, col++].Value = "Date Update"; sheet.Cells[row, col++].Value = "Latitude"; sheet.Cells[row, col++].Value = "Longitude"; sheet.Cells[row, col++].Value = "City"; sheet.Cells[row, col++].Value = "Region"; var descriptionDataCol = col; sheet.Cells[row, col++].Value = "Address"; //sheet.Cells[row, col++].Value = "Description"; sheet.Cells[row, col++].Value = "Floor"; sheet.Cells[row, col++].Value = "Size"; sheet.Cells[row, col++].Value = "AgencyName"; sheet.Cells[row, col++].Value = "Contact"; sheet.Cells[row, col++].Value = "Phone"; sheet.Cells[row, col++].Value = "Phone1"; sheet.Cells[row, col++].Value = "Phone2"; sheet.Cells[row, col++].Value = "סורגים"; sheet.Cells[row, col++].Value = "לשותפים"; sheet.Cells[row, col++].Value = "ריהוט"; sheet.Cells[row, col++].Value = "מעלית"; sheet.Cells[row, col++].Value = "מרפסת"; sheet.Cells[row, col++].Value = "חניה"; sheet.Cells[row, col++].Value = "מזגן"; amountDataCols = col; sheet.Cells[row, col++].Value = "Link"; sheet.Cells[row, 1, row, col - 1].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center; sheet.Cells[1, 1, 1, amountDataCols - 1].AutoFilter = true; row++; col = 1; foreach (var item in items) { col = 1; sheet.Cells[row, col++].Value = item.ItemId; _addCellDate(sheet.Cells[row, col], item.DateUpdated); col++; sheet.Cells[row, col++].Value = item.Latitude; sheet.Cells[row, col++].Value = item.Longitude; sheet.Cells[row, col++].Value = item.City; sheet.Cells[row, col++].Value = item.Region; sheet.Cells[row, col++].Value = item.Address; //sheet.Cells[row, col++].Value = item.Description; _addCellInt(sheet.Cells[row, col], item.Floor); col++; _addCellInt(sheet.Cells[row, col], item.Size); col++; sheet.Cells[row, col++].Value = item.AgencyName; sheet.Cells[row, col++].Value = item.Contact; sheet.Cells[row, col++].Value = item.Phone; sheet.Cells[row, col++].Value = item.Phone1; sheet.Cells[row, col++].Value = item.Phone2; sheet.Cells[row, col++].Value = item.Field0; sheet.Cells[row, col++].Value = item.Field1; sheet.Cells[row, col++].Value = item.Field2; sheet.Cells[row, col++].Value = item.Field3; sheet.Cells[row, col++].Value = item.Field4; sheet.Cells[row, col++].Value = item.Field5; sheet.Cells[row, col++].Value = item.Field6; // link var url = $"{item.LinkToProfile}"; sheet.Cells[row, col].Value = url; sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center; sheet.Cells[row, col].Hyperlink = new Uri(url); col++; var i = 1; foreach (var image in item.Images) { _addCellLinks(sheet.Cells[row, col], image.Full, i++); col++; } if (item.Images.Count > hasAmountImages) { hasAmountImages = item.Images.Count; } row++; } col--; using (var cells = sheet.Cells[sheet.Cells[1, 1, 1 + items.Count, amountDataCols + hasAmountImages].Address]) { cells.Style.Border.Top.Style = ExcelBorderStyle.Thin; cells.Style.Border.Bottom.Style = ExcelBorderStyle.Thin; cells.Style.Border.Right.Style = ExcelBorderStyle.Thin; cells.Style.Border.Left.Style = ExcelBorderStyle.Thin; cells.AutoFitColumns(); } sheet.Column(descriptionDataCol).Width = 50; var startPositionOnFileLinks = amountDataCols + 1; var endPositionOnFileLinks = startPositionOnFileLinks + hasAmountImages - 1; foreach (var i in Enumerable.Range(startPositionOnFileLinks, hasAmountImages)) { //sheet.Column(i).Width = 5; sheet.Cells[1, i].Value = $"Images {i - startPositionOnFileLinks + 1}"; } foreach (var i in Enumerable.Range(2, row)) { sheet.Row(i).Height = 15; } result = new MemoryStream(eP.GetAsByteArray()); } _log("Excel data create done"); return(result); }