public void MarkSold(string systemName)
{
using (var crawler = new Crawler())
{
_lock = new object();
lock (_lock)
{
using (var db = new Funda.WebAnalyzerEntities())
{
var now = DateTime.Now;
var list = new List <IRecord>();
list = db.Rent.Where <IRecord>(o => (!o.DateRemoved.HasValue) /* || o.DateLastProcessed.Value < DateTime.Today) && o.DateRemoved == null */ && o.Url.Contains(systemName)).ToList().Union(db.Sale.Where <IRecord>(o => (!o.DateRemoved.HasValue) && o.Url.Contains(systemName)).ToList()).ToList();
foreach (var rent in list)
{
try
{
crawler.Navigate(rent.Url);
if (systemName == "fotocasa")
{
crawler.MarkSoldFotoCasa(rent);
}
db.SaveChanges();
}
catch { }
}
}
}
}
}
public ActionResult CollectNewIe()
{
using (var crawler = new Funda.Crawler())
{
using (var db = new Funda.WebAnalyzerEntities())
{
foreach (var search in DaftieSearchList())
{
// SetMinMax(search);
for (int i = 0; i < 30; i++)
{
try
{
search.PaginationNumber = i;
crawler.Navigate(search);
var adverts = Enumerable.Empty <IRecord>();
if (search.IsSale)
{
adverts = crawler.AddDaft(true).Where(o => o.Price != null).ExceptWhere(db.Sale, o => o.Url);
db.Sale.AddRange(adverts.Cast <Sale>().ToList());
}
else
{
adverts = crawler.AddDaft(false).Where(o => o.Price != null).ExceptWhere(db.Rent, o => o.Url);
db.Rent.AddRange(adverts.Cast <Rent>().ToList());
}
foreach (var advert in adverts)
{
advert.Title = search.Text.Split('-')[0];
}
if (!adverts.Any())
{
break;
}
db.SaveChanges();
}
catch
{
}
}
}
}
}
return(RedirectToAction("UpdateDaft"));
}
public ActionResult CollectNewMestoUe()
{
using (var crawler = new Funda.Crawler())
{
using (var db = new Funda.WebAnalyzerEntities())
{
foreach (var search in MestoUaSearchList())
{
// SetMinMax(search);
for (int i = 1; i < 15; i++)
{
try
{
search.PaginationNumber = i;
crawler.Navigate(search);
var adverts = Enumerable.Empty <IRecord>();
if (search.IsSale)
{
adverts = crawler.AddNewMestoUeSales((Crawler.MestoUaSearch)search).Where(o => o.Price != null).ExceptWhere(db.Sale, o => o.Url);
db.Sale.AddRange(adverts.Cast <Sale>().ToList());
}
else
{
adverts = crawler.AddNewLtRents().Where(o => o.Price != null).ExceptWhere(db.Rent, o => o.Url);
db.Rent.AddRange(adverts.Cast <Rent>().ToList());
}
if (!adverts.Any())
{
break;
}
db.SaveChanges();
}
catch
{
}
}
}
}
}
return(RedirectToAction("UpdateExistingLt"));
}
public ActionResult CollectNewEs()
{
using (var crawler = new Funda.Crawler())
{
using (var db = new Funda.WebAnalyzerEntities())
{
foreach (var search in FotoCasaSearchList())
{
// SetMinMax(search);
for (int i = 1; i < 15; i++)
{
try
{
search.PaginationNumber = i;
crawler.Navigate(search);
var adverts = Enumerable.Empty <IRecord>();
if (search.IsSale)
{
adverts = crawler.AddFotoCasaSales().Where(o => o.Price != null).ExceptWhere(db.Sale, o => o.Url);
db.Sale.AddRange(adverts.Cast <Sale>().ToList());
}
else
{
adverts = crawler.AddFotoCasaRents().Where(o => o.Price != null).ExceptWhere(db.Rent, o => o.Url);
db.Rent.AddRange(adverts.Cast <Rent>().ToList());
}
if (!adverts.Any())
{
break;
}
db.SaveChanges();
}
catch
{
}
}
}
//foreach (var search in PisosSearchList())
//{
// for (int i = 1; i < 3; i++)
// {
// try
// {
// search.PaginationNumber = i;
// crawler.Navigate(search);
// var adverts = Enumerable.Empty<IRecord>();
// if (search.IsSale)
// {
// adverts = crawler.AddPisosSales().Where(o => o.Price != null).ExceptWhere(db.Sale, o => o.Url);
// db.Sale.AddRange(adverts.Cast<Sale>().ToList());
// }
// else
// {
// adverts = crawler.AddPisosRents().Where(o => o.Price != null).ExceptWhere(db.Rent, o => o.Url);
// db.Rent.AddRange(adverts.Cast<Rent>().ToList());
// }
// if (!adverts.Any())
// {
// break;
// }
// db.SaveChanges();
// }
// catch
// {
// }
// }
//}
}
}
return(RedirectToAction("UpdateFotoCasa"));
}
public void DoUpdate(string systemName, bool?isSale)
{
using (var crawler = new Crawler())
{
_lock = new object();
lock (_lock)
{
using (var db = new Funda.WebAnalyzerEntities())
{
var now = DateTime.Now;
var list = new List <IRecord>();
//if (!isSale.HasValue)
//{
list = db.Rent.Where <IRecord>(o => (!o.DateLastProcessed.HasValue || !o.DateRemoved.HasValue) /* || o.DateLastProcessed.Value < DateTime.Today) && o.DateRemoved == null */ && o.Url.Contains(systemName)).ToList().Union(
db.Sale.Where <IRecord>(o => (!o.DateLastProcessed.HasValue || !o.DateRemoved.HasValue) && o.Url.Contains(systemName)).ToList()
)
.OrderBy(o => o.DateLastProcessed).ToList();
//}
//else
//{
// if (isSale.Value)
// {
// list = db.Sale.Where<IRecord>(o => (!o.DateLastProcessed.HasValue || o.DateLastProcessed.Value < DateTime.Today) && o.DateRemoved == null && o.Url.Contains(systemName)).ToList();
// }
// else
// {
// list = db.Rent.Where<IRecord>(o => (!o.DateLastProcessed.HasValue || o.DateLastProcessed.Value < DateTime.Today) && o.DateRemoved == null && o.Url.Contains(systemName)).ToList();
// }
//}
foreach (var record in list)
{
try
{
crawler.Navigate(record.Url);
if (systemName == "funda")
{
crawler.GetRecordDataFromItsPage(record);
}
else if (systemName == "fotocasa")
{
crawler.GetRecordDataFromFotoCasa(record);
}
else if (systemName == "daft")
{
if (record is Sale)
{
crawler.GetRecordDataFromDaftSale(record);
}
else
{
crawler.GetRecordDataFromDaftRent(record);
}
}
else
{
crawler.GetRecordDataFromItsPageLt(record);
}
db.SaveChanges();
}
catch { }
}
}
}
}
}
