public bool UpdateData(DataDomainModel data)
{
var result = true;
Truncate();
var items = (List <AdItemWinWinDomainModel>)data.Data;
var index = 0;
foreach (var item in items)
{
var itemDb = new AdItemWinWinDbModel().FromDomain(item);
_context.DataWinWin.Add(itemDb);
index++;
if (index % 100 == 0)
{
Console.WriteLine($"index={index}");
_context.SaveChanges();
}
}
_context.SaveChanges();
UpdateGeometry();
return(result);
}
public bool UpdateData(DataDomainModel data)
{
var result = true;
Truncate();
var items = (List <AdItemHomeLessDomainModel>)data.Data;
Console.WriteLine($"Need update items: {items.Count()}");
var index = 0;
foreach (var item in items)
{
if (item.IsValidForDbModel)
{
var itemDb = new AdItemHomeLessDbModel().FromDomain(item);
_context.DataHomeLess.Add(itemDb);
index++;
if (index % 300 == 0)
{
Console.WriteLine($"index={index}");
_context.SaveChanges();
}
}
}
_context.SaveChanges();
UpdateGeometry();
return(result);
}
public DataDomainModel SaveDomainModel(DataDomainModel model)
{
var state = _state;
//var model = GetDomainModelAsync().Result;
var filename = $"{state.RootPath}/domain-model.json";
File.WriteAllText(filename, JsonConvert.SerializeObject(model, Formatting.Indented));
return(model);
}
/// <summary>
/// Deletes the domain.
/// </summary>
/// <param name="id">The domain's id.</param>
/// <returns>Refresh the page.</returns>
public async Task <IActionResult> OnPostDeleteAsync(int id)
{
DomainData domainData = new DomainData(_db);
DataDomainModel newDomainModel = new DataDomainModel
{
Domain = DisplayDomain.Domain,
Id = DisplayDomain.Id,
};
await domainData.DeleteDomain(newDomainModel);
return(RedirectToPage("./domain"));
}
public DataDomainModel GetDataFromScraper(RequestDataScrape request)
{
var result = new DataDomainModel();
switch (request.ScrapeId)
{
case EnumScrapers.Airdna:
var scraper = new AirdnaScraper();
result = scraper.Scrape(request.CityId);
break;
}
return(result);
}
public override async Task <DataDomainModel> GetDomainModelAsync()
{
var state = (ScraperHomeLessStateModel)_state;
SetWorkPhaseBase($"DomainModel", state);
var listDomainItems = ScrapePhase4Async(state);
var result = new DataDomainModel()
{
Scraper = state.TypeScraper,
Data = await listDomainItems,
};
LogDone(state);
return(result);
}
public async override Task <DataDomainModel> GetDomainModelAsync()
{
var state = (ScraperOnmapStateModel)_state;
SetWorkPhaseBase($"DomainModel", state);
var domainModel = ScrapePhase3Async(state);
var result = new DataDomainModel()
{
Scraper = EnumScrapers.Onmap,
Data = await domainModel,
};
LogDone(state);
return(result);
}
public override async Task <DataDomainModel> GetDomainModelAsync()
{
var state = (ScraperYad2StateModel)_state;
SetWorkPhaseBase($"DomainModel", state);
var domainModel = _scrapePhase7_GenerateDomainModelAsync(state);
DataDomainModel result = new DataDomainModel()
{
Scraper = EnumScrapers.Yad2,
Data = await domainModel,
};
LogDone(state);
return(result);
}
public async override Task <DataDomainModel> GetDomainModelAsync()
{
var state = _state;
SetWorkPhaseBase($"DomainModel", state);
var listDomainItems = await ScrapePhase_GetDomainModelAsync(state);
var result = new DataDomainModel()
{
Scraper = state.TypeScraper,
Data = listDomainItems,
};
_logBase($"DomainModel has {listDomainItems.Count()} AdItems", state);
LogDone(state);
return(result);
}
/// <summary>
/// Creates a new domain.
/// </summary>
/// <returns>Refresh the page.</returns>
public async Task <IActionResult> OnPost()
{
if (!ModelState.IsValid)
{
return(Page());
}
DomainData domainData = new DomainData(_db);
DataDomainModel newDomainModel = new DataDomainModel
{
Domain = DisplayDomain.Domain,
Id = DisplayDomain.Id,
};
await domainData.InsertDomain(newDomainModel);
return(RedirectToPage("./domain"));
}
public MemoryStream CreateExcel(DataDomainModel data)
{
MemoryStream result = null;
var itemsDomainModel = (List <AdItemYad2DomainModel>)data.Data;
var items = new List <AdItemYad2ExcelModel>();
foreach (var item in itemsDomainModel)
{
items.Add(new AdItemYad2ExcelModel().FromDomain(item));
}
var amountDataCols = 0;
var hasAmountImages = 1;
_log($"Amount input items: {items.Count}");
using (ExcelPackage eP = new ExcelPackage())
{
eP.Workbook.Properties.Author = "Scrap";
eP.Workbook.Properties.Title = "Scrap Data";
eP.Workbook.Properties.Company = "Yad2";
var sheet = eP.Workbook.Worksheets.Add($"Yad2");
var row = 1;
var col = 1;
// head line
sheet.Cells[row, col++].Value = "ItemId";
sheet.Cells[row, col++].Value = "Date Create";
sheet.Cells[row, col++].Value = "Date Update";
sheet.Cells[row, col++].Value = "Latitude";
sheet.Cells[row, col++].Value = "Longitude";
sheet.Cells[row, col++].Value = "City";
sheet.Cells[row, col++].Value = "HouseNumber";
sheet.Cells[row, col++].Value = "Neighborhood";
sheet.Cells[row, col++].Value = "StreetName";
sheet.Cells[row, col++].Value = "AriaBase";
sheet.Cells[row, col++].Value = "Balconies";
sheet.Cells[row, col++].Value = "Pets";
sheet.Cells[row, col++].Value = "Elevators";
sheet.Cells[row, col++].Value = "FloorOn";
sheet.Cells[row, col++].Value = "FloorOf";
sheet.Cells[row, col++].Value = "Rooms";
sheet.Cells[row, col++].Value = "Parking";
//sheet.Cells[row, col++].Value = "ContactEmail";
sheet.Cells[row, col++].Value = "ContactName";
sheet.Cells[row, col++].Value = "ContactPhone";
sheet.Cells[row, col++].Value = "Price";
var descriptionDataCol = col;
sheet.Cells[row, col++].Value = "Description";
sheet.Cells[row, col++].Value = "PropertyType";
sheet.Cells[row, col++].Value = "AirConditioner";
amountDataCols = col;
sheet.Cells[row, col++].Value = "Link";
//sheet.Cells[row, 1, row, 23].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
sheet.Cells[1, 1, 1, amountDataCols - 1].AutoFilter = true;
row++; col = 1;
foreach (var item in items)
{
col = 1;
sheet.Cells[row, col++].Value = item.ItemId;
_addCellDate(sheet.Cells[row, col], item.DateCreate); col++;
_addCellDate(sheet.Cells[row, col], item.DateUpdate); col++;
sheet.Cells[row, col++].Value = item.Latitude;
sheet.Cells[row, col++].Value = item.Longitude;
sheet.Cells[row, col++].Value = item.HeCity;
_addCellInt(sheet.Cells[row, col], item.HeHouseNumber); col++;
sheet.Cells[row, col++].Value = item.HeNeighborhood;
sheet.Cells[row, col++].Value = item.HeStreetName;
_addCellInt(sheet.Cells[row, col], item.AriaBase); col++;
_addCellInt(sheet.Cells[row, col], item.Balconies); col++;
sheet.Cells[row, col++].Value = item.Pets;
sheet.Cells[row, col++].Value = item.Elevators;
_addCellInt(sheet.Cells[row, col], item.FloorOn); col++;
_addCellInt(sheet.Cells[row, col], item.FloorOf); col++;
_addCellInt(sheet.Cells[row, col], item.Rooms); col++;
sheet.Cells[row, col++].Value = item.Parking;
//sheet.Cells[row, col++].Value = item.ContactEmail;
sheet.Cells[row, col++].Value = item.ContactName;
sheet.Cells[row, col++].Value = item.ContactPhone;
sheet.Cells[row, col++].Value = item.Price;
sheet.Cells[row, col++].Value = item.Description;
sheet.Cells[row, col++].Value = item.PropertyType;
sheet.Cells[row, col++].Value = item.AirConditioner;
// link
var url = $"https://www.yad2.co.il/item/{item.ItemId}";
sheet.Cells[row, col].Value = url;
sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
sheet.Cells[row, col].Hyperlink = new Uri(url);
col++;
if (item?.Images != null)
{
var i = 1;
foreach (var image in item.Images)
{
_addCellLinks(sheet.Cells[row, col], image, i++);
col++;
}
if (item.Images.Count > hasAmountImages)
{
hasAmountImages = item.Images.Count;
}
}
row++;
}
col--;
using (var cells = sheet.Cells[sheet.Cells[1, 1, 1 + items.Count, amountDataCols + hasAmountImages].Address])
{
cells.Style.Border.Top.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Bottom.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Right.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Left.Style = ExcelBorderStyle.Thin;
cells.AutoFitColumns();
}
sheet.Column(descriptionDataCol).Width = 50;
var startPositionOnFileLinks = amountDataCols + 1;
var endPositionOnFileLinks = startPositionOnFileLinks + hasAmountImages;
foreach (var i in Enumerable.Range(startPositionOnFileLinks, hasAmountImages))
{
sheet.Cells[1, i].Value = $"Images {i - startPositionOnFileLinks + 1}";
}
foreach (var i in Enumerable.Range(3, row))
{
sheet.Row(i).Height = 15;
}
result = new MemoryStream(eP.GetAsByteArray());
}
return(result);
}
public MemoryStream CreateExcel(DataDomainModel data)
{
_log($"Start create excel data");
MemoryStream result = null;
var itemsDomainModel = (List <AdItemWinWinDomainModel>)data.Data;
var items = new List <AdItemWinWinExcelModel>();
foreach (var item in itemsDomainModel)
{
items.Add(new AdItemWinWinExcelModel().FromDomain(item));
}
var amountDataCols = 0;
var hasAmountImages = 1;
_log($"Amount input items: {items.Count}");
using (ExcelPackage eP = new ExcelPackage())
{
eP.Workbook.Properties.Author = "Scrap";
eP.Workbook.Properties.Title = "Scrap Data";
eP.Workbook.Properties.Company = "WinWin";
var sheet = eP.Workbook.Worksheets.Add($"WinWin");
var row = 1;
var col = 1;
// head line
sheet.Cells[row, col++].Value = "ItemId";
sheet.Cells[row, col++].Value = "Date Update";
sheet.Cells[row, col++].Value = "Longitude";
sheet.Cells[row, col++].Value = "Latitude";
sheet.Cells[row, col++].Value = "City";
sheet.Cells[row, col++].Value = "Area";
sheet.Cells[row, col++].Value = "StreetAddress";
sheet.Cells[row, col++].Value = "Rooms";
sheet.Cells[row, col++].Value = "Floor";
sheet.Cells[row, col++].Value = "Status";
sheet.Cells[row, col++].Value = "DateEnter";
sheet.Cells[row, col++].Value = "Square";
sheet.Cells[row, col++].Value = "IsPartners";
sheet.Cells[row, col++].Value = "AmountPayment";
var descriptionDataCol = col;
sheet.Cells[row, col++].Value = "Description";
sheet.Cells[row, col++].Value = "Price";
sheet.Cells[row, col++].Value = "IsAgent";
sheet.Cells[row, col++].Value = "ContactName";
sheet.Cells[row, col++].Value = "Phone1";
sheet.Cells[row, col++].Value = "Phone2";
amountDataCols = col;
sheet.Cells[row, col++].Value = "Link";
//sheet.Cells[row, 1, row, col - 1].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
sheet.Cells[1, 1, 1, amountDataCols - 1].AutoFilter = true;
row++; col = 1;
foreach (var item in items)
{
col = 1;
sheet.Cells[row, col++].Value = item.ItemId;
_addCellDate(sheet.Cells[row, col], item.DateUpdate); col++;
sheet.Cells[row, col++].Value = item.Latitude;
sheet.Cells[row, col++].Value = item.Longitude;
sheet.Cells[row, col++].Value = item.City;
_addCellInt(sheet.Cells[row, col], item.Area); col++;
sheet.Cells[row, col++].Value = item.StreetAddress;
_addCellInt(sheet.Cells[row, col], item.Rooms); col++;
_addCellInt(sheet.Cells[row, col], item.Floor); col++;
sheet.Cells[row, col++].Value = item.State;
_addCellDate(sheet.Cells[row, col], item.DateEnter); col++;
_addCellInt(sheet.Cells[row, col], item.Square); col++;
sheet.Cells[row, col++].Value = item.IsPartners;
sheet.Cells[row, col++].Value = item.AmountPayment;
sheet.Cells[row, col++].Value = item.Description;
_addCellInt(sheet.Cells[row, col], item.Price); col++;
sheet.Cells[row, col++].Value = item.IsAgent;
sheet.Cells[row, col++].Value = item.ContactName;
sheet.Cells[row, col++].Value = item.Phone1;
sheet.Cells[row, col++].Value = item.Phone2;
// link
var url = $"https://www.winwin.co.il/RealEstate/ForRent/Ads/RealEstateAds,{item.ItemId}.aspx";
sheet.Cells[row, col].Value = url;
sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
sheet.Cells[row, col].Hyperlink = new Uri(url);
col++;
var i = 1;
foreach (var image in item.Images)
{
_addCellLinks(sheet.Cells[row, col], image.Full, i++);
col++;
}
if (item.Images.Count > hasAmountImages)
{
hasAmountImages = item.Images.Count;
}
row++;
}
col--;
using (var cells = sheet.Cells[sheet.Cells[1, 1, 1 + items.Count, amountDataCols + hasAmountImages].Address])
{
cells.Style.Border.Top.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Bottom.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Right.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Left.Style = ExcelBorderStyle.Thin;
cells.AutoFitColumns();
}
sheet.Column(descriptionDataCol).Width = 50;
var startPositionOnFileLinks = amountDataCols + 1;
var endPositionOnFileLinks = startPositionOnFileLinks + hasAmountImages;
foreach (var i in Enumerable.Range(startPositionOnFileLinks, hasAmountImages))
{
sheet.Cells[1, i].Value = $"Images {i - startPositionOnFileLinks + 1}";
}
foreach (var i in Enumerable.Range(3, row))
{
sheet.Row(i).Height = 15;
}
result = new MemoryStream(eP.GetAsByteArray());
}
_log("Excel data create done");
return(result);
}
public MemoryStream CreateExcel(DataDomainModel data)
{
_log($"Start create excel data");
MemoryStream result = null;
var itemsDomainModel = (List <AdItemAirdnaDomainModel>)data.Data;
var items = new List <AdItemAirdnaExcelModel>();
foreach (var item in itemsDomainModel)
{
items.Add(new AdItemAirdnaExcelModel().FromDomain(item));
}
_log($"Amount input items: {items.Count}");
using (ExcelPackage eP = new ExcelPackage())
{
eP.Workbook.Properties.Author = "Scrap";
eP.Workbook.Properties.Title = "Scrap Data";
eP.Workbook.Properties.Company = "Airdna";
var sheet = eP.Workbook.Worksheets.Add($"Airdna");
var row = 1;
var col = 1;
// head line
var descriptionDataCol = col;
sheet.Cells[row, col++].Value = "Title (link to site)";
sheet.Cells[row, col++].Value = "Location";
sheet.Cells[row, col++].Value = "Latitude";
sheet.Cells[row, col++].Value = "Longitude";
sheet.Cells[row, col++].Value = "Adr";
sheet.Cells[row, col++].Value = "Rating";
sheet.Cells[row, col++].Value = "Bathrooms";
sheet.Cells[row, col++].Value = "Bedrooms";
sheet.Cells[row, col++].Value = "Accommodates";
sheet.Cells[row, col++].Value = "Revenue";
sheet.Cells[row, col++].Value = "Property Type";
sheet.Cells[row, col++].Value = "Occ";
sheet.Cells[row, col++].Value = "Reviews";
sheet.Cells[row, col++].Value = "RoomType";
sheet.Cells[row, col++].Value = "Link";
row++; col = 1;
foreach (var item in items)
{
col = 1;
sheet.Cells[row, col++].Value = item.Title;
sheet.Cells[row, col++].Value = item.Location;
sheet.Cells[row, col++].Value = item.Latitude;
sheet.Cells[row, col++].Value = item.Longitude;
sheet.Cells[row, col++].Value = item.Adr;
sheet.Cells[row, col++].Value = item.Rating;
sheet.Cells[row, col++].Value = item.Bathrooms;
sheet.Cells[row, col++].Value = item.Bedrooms;
sheet.Cells[row, col++].Value = item.Accommodates;
sheet.Cells[row, col++].Value = item.Revenue;
sheet.Cells[row, col++].Value = item.PropertyType;
sheet.Cells[row, col++].Value = item.Occ;
sheet.Cells[row, col++].Value = item.Reviews;
sheet.Cells[row, col++].Value = item.RoomType;
// link
sheet.Cells[row, col].Value = item.LinkToProfile;
sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
sheet.Cells[row, col].Hyperlink = new Uri(item.LinkToProfile);
col++;
row++;
}
col--;
sheet.Column(descriptionDataCol).Width = 50;
result = new MemoryStream(eP.GetAsByteArray());
}
_log("Excel data create done");
return(result);
}
public MemoryStream CreateExcel(DataDomainModel data)
{
_log($"Start create excel data");
MemoryStream result = null;
var itemsDomainModel = (List <AdItemHomeLessDomainModel>)data.Data;
var items = new List <AdItemHomeLessExcelModel>();
foreach (var item in itemsDomainModel)
{
items.Add(new AdItemHomeLessExcelModel().FromDomain(item));
}
var amountDataCols = 0;
var hasAmountImages = 1;
_log($"Amount input items: {items.Count}");
using (ExcelPackage eP = new ExcelPackage())
{
eP.Workbook.Properties.Author = "Scrap";
eP.Workbook.Properties.Title = "Scrap Data";
eP.Workbook.Properties.Company = "HomeLess";
var sheet = eP.Workbook.Worksheets.Add($"HomeLess");
var row = 1;
var col = 1;
// head line
sheet.Cells[row, col++].Value = "ItemId";
sheet.Cells[row, col++].Value = "Date Update";
sheet.Cells[row, col++].Value = "Latitude";
sheet.Cells[row, col++].Value = "Longitude";
sheet.Cells[row, col++].Value = "City";
sheet.Cells[row, col++].Value = "Region";
var descriptionDataCol = col;
sheet.Cells[row, col++].Value = "Address";
//sheet.Cells[row, col++].Value = "Description";
sheet.Cells[row, col++].Value = "Floor";
sheet.Cells[row, col++].Value = "Size";
sheet.Cells[row, col++].Value = "AgencyName";
sheet.Cells[row, col++].Value = "Contact";
sheet.Cells[row, col++].Value = "Phone";
sheet.Cells[row, col++].Value = "Phone1";
sheet.Cells[row, col++].Value = "Phone2";
sheet.Cells[row, col++].Value = "סורגים";
sheet.Cells[row, col++].Value = "לשותפים";
sheet.Cells[row, col++].Value = "ריהוט";
sheet.Cells[row, col++].Value = "מעלית";
sheet.Cells[row, col++].Value = "מרפסת";
sheet.Cells[row, col++].Value = "חניה";
sheet.Cells[row, col++].Value = "מזגן";
amountDataCols = col;
sheet.Cells[row, col++].Value = "Link";
sheet.Cells[row, 1, row, col - 1].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
sheet.Cells[1, 1, 1, amountDataCols - 1].AutoFilter = true;
row++; col = 1;
foreach (var item in items)
{
col = 1;
sheet.Cells[row, col++].Value = item.ItemId;
_addCellDate(sheet.Cells[row, col], item.DateUpdated); col++;
sheet.Cells[row, col++].Value = item.Latitude;
sheet.Cells[row, col++].Value = item.Longitude;
sheet.Cells[row, col++].Value = item.City;
sheet.Cells[row, col++].Value = item.Region;
sheet.Cells[row, col++].Value = item.Address;
//sheet.Cells[row, col++].Value = item.Description;
_addCellInt(sheet.Cells[row, col], item.Floor); col++;
_addCellInt(sheet.Cells[row, col], item.Size); col++;
sheet.Cells[row, col++].Value = item.AgencyName;
sheet.Cells[row, col++].Value = item.Contact;
sheet.Cells[row, col++].Value = item.Phone;
sheet.Cells[row, col++].Value = item.Phone1;
sheet.Cells[row, col++].Value = item.Phone2;
sheet.Cells[row, col++].Value = item.Field0;
sheet.Cells[row, col++].Value = item.Field1;
sheet.Cells[row, col++].Value = item.Field2;
sheet.Cells[row, col++].Value = item.Field3;
sheet.Cells[row, col++].Value = item.Field4;
sheet.Cells[row, col++].Value = item.Field5;
sheet.Cells[row, col++].Value = item.Field6;
// link
var url = $"{item.LinkToProfile}";
sheet.Cells[row, col].Value = url;
sheet.Cells[row, col].Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
sheet.Cells[row, col].Hyperlink = new Uri(url);
col++;
var i = 1;
foreach (var image in item.Images)
{
_addCellLinks(sheet.Cells[row, col], image.Full, i++);
col++;
}
if (item.Images.Count > hasAmountImages)
{
hasAmountImages = item.Images.Count;
}
row++;
}
col--;
using (var cells = sheet.Cells[sheet.Cells[1, 1, 1 + items.Count, amountDataCols + hasAmountImages].Address])
{
cells.Style.Border.Top.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Bottom.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Right.Style = ExcelBorderStyle.Thin;
cells.Style.Border.Left.Style = ExcelBorderStyle.Thin;
cells.AutoFitColumns();
}
sheet.Column(descriptionDataCol).Width = 50;
var startPositionOnFileLinks = amountDataCols + 1;
var endPositionOnFileLinks = startPositionOnFileLinks + hasAmountImages - 1;
foreach (var i in Enumerable.Range(startPositionOnFileLinks, hasAmountImages))
{
//sheet.Column(i).Width = 5;
sheet.Cells[1, i].Value = $"Images {i - startPositionOnFileLinks + 1}";
}
foreach (var i in Enumerable.Range(2, row))
{
sheet.Row(i).Height = 15;
}
result = new MemoryStream(eP.GetAsByteArray());
}
_log("Excel data create done");
return(result);
}
