public Model.SearchResult AddSearchResult(int parsedAdvertismentsCount = 0)
{
var advertSection = (from section in _dbcontext.AdvertismentSections
where section.code == _sectionCode
select section).SingleOrDefault();
if (advertSection == null)
{
throw new Exception("Section name with code '" + _sectionCode + "' was not founded");
}
var searchResults = from lastSearchResultFromDB in _dbcontext.SearchResults
where lastSearchResultFromDB.AdvertismentSection.code == _sectionCode
orderby lastSearchResultFromDB.createDate descending
select lastSearchResultFromDB;
Model.SearchResult searchResult;
Model.SearchResult lastSearchResult = null;
if (searchResults.Any())
{
lastSearchResult = searchResults.Take(1).SingleOrDefault();
}
if (lastSearchResult != null &&
lastSearchResult.createDate.Date == Utils.GetUkranianDateTimeNow().Date)
{
lastSearchResult.modifyDate = Utils.GetUkranianDateTimeNow();
lastSearchResult.allParsedAdvertismentsCount = parsedAdvertismentsCount;
searchResult = lastSearchResult;
}
else
{
searchResult = new Model.SearchResult
{
createDate = Utils.GetUkranianDateTimeNow(),
modifyDate = Utils.GetUkranianDateTimeNow(),
AdvertismentSection = advertSection,
allParsedAdvertismentsCount = parsedAdvertismentsCount
};
_dbcontext.SearchResults.InsertOnSubmit(searchResult);
}
_dbcontext.SubmitChanges();
return(searchResult);
}
public int Crawl(IList <SiteSetting> siteSettings)
{
//-- create new SearchResult
var searchResults = new SearchResults(sectionCode, context);
Model.SearchResult searchResult = searchResults.AddSearchResult();
var advertisments = new List <Server.Entities.Advertisment>();
foreach (SiteSetting siteSetting in siteSettings)
{
try
{
GetAdvertisments(siteSetting, ref advertisments);
PrepairAdvertisments(sectionCode, context, ref advertisments);
}
catch (Exception e)
{
if (e is ThreadAbortException)
{
WriteLog("Crawler ThreadAbortException.");
}
WriteLog("Site parser full error." + Environment.NewLine +
"Site: " + siteSetting.name + Environment.NewLine
+ "Error: " + e.Message + Environment.NewLine
+ "Trace:" + e.StackTrace);
if (e.InnerException != null)
{
WriteLog("Inner exception: " + e.InnerException.Message);
}
}
}
searchResult.allParsedAdvertismentsCount = SaveAdvertisments(searchResult, advertisments);
context.SubmitChanges();
return(advertisments.Count);
}
private int SaveAdvertisments(Model.SearchResult searchResult, IList <Server.Entities.Advertisment> advertisments)
{
Utils.PingServer();
WriteLog("Saving advertisments in DB.");
int savedAdvertismentsCount = 0;
foreach (var advertisment in advertisments)
{
try
{
var textHash = Utils.HashToHex(Utils.CalculateMD5Hash(advertisment.Text), true);
Model.Advertisment existsAdvertisment = context.Advertisments
.Where(a => a.TextHashValue == textHash)
.FirstOrDefault();
if (existsAdvertisment == null)
{
#region New Advertisment
Model.AdvertismentSubSection subSectionObject = null;
if (advertisment.SubSectionID != null)
{
subSectionObject = context.AdvertismentSubSections
.SingleOrDefault(s => s.Id == advertisment.SubSectionID.Value);
if (subSectionObject == null)
{
throw new Exception("Can not find Sub Section. ID: " + advertisment.SubSectionID.Value);
}
}
var advertismentEntity = new Model.Advertisment
{
createDate = Utils.GetUkranianDateTimeNow(),
modifyDate = Utils.GetUkranianDateTimeNow(),
text = advertisment.Text,
AdvertismentSection = searchResult.AdvertismentSection,
SearchResult = searchResult,
link = advertisment.Link,
siteName = advertisment.SiteName,
subpurchaseAdvertisment = true,
AdvertismentSubSection = subSectionObject,
Price = advertisment.Price,
Address1 = advertisment.Address1,
UndergroundStationID = advertisment.UndergroundStationID,
City = advertisment.City
};
context.Advertisments.InsertOnSubmit(advertismentEntity);
//try
//{
// context.SubmitChanges();
//}
//catch (Exception e)
//{
// WriteLog("Saving advertisment error. " + Environment.NewLine
// + "Text: " + advertisment.Text + Environment.NewLine
// + "Link: " + advertisment.Link + Environment.NewLine
// + "Site: " + advertisment.SiteName + Environment.NewLine
// + "Error: " + e.Message + Environment.NewLine
// + ". Trace:" + e.StackTrace);
// throw;
//}
//--- add phones
foreach (var phone in advertisment.Phones)
{
if (!string.IsNullOrWhiteSpace(phone))
{
//_log.WriteLog(phone.phone);
var advertismentPhoneEntity = new Model.AdvertismentPhone
{
phone = phone,
Advertisment = advertismentEntity
};
context.AdvertismentPhones.InsertOnSubmit(advertismentPhoneEntity);
//advertismentEntity.AdvertismentPhones.Add(advertismentPhoneEntity);
}
}
//----
//--- add photos
foreach (var photoUrl in advertisment.PhotoUrls)
{
if (!string.IsNullOrWhiteSpace(photoUrl))
{
var advertismentPhotoEntity = new Model.AdvertismentsPhoto
{
filename = photoUrl,
createDate = Utils.GetUkranianDateTimeNow(),
Advertisment = advertismentEntity
};
context.AdvertismentsPhotos.InsertOnSubmit(advertismentPhotoEntity);
//advertismentEntity.AdvertismentsPhotos.Add(advertismentPhotoEntity);
}
}
//---
try
{
context.SubmitChanges();
}
catch (Exception e)
{
WriteLog("Saving advertisment's error. " + Environment.NewLine
+ "Error: " + e.Message + Environment.NewLine
+ ". Trace:" + e.StackTrace);
throw;
}
savedAdvertismentsCount++;
#endregion New Advertisment
}
else
{
var advertismentUpdate = new Model.AdvertismentUpdate()
{
AdvertismentUpdateID = Guid.NewGuid(),
AdvertismentID = existsAdvertisment.Id,
CreateDate = Utils.GetUkranianDateTimeNow(),
SearchResultID = searchResult.Id
};
context.AdvertismentUpdates.InsertOnSubmit(advertismentUpdate);
//--- update modified datetime for advertisment
var modifyAdvertisment = context.Advertisments
.FirstOrDefault(a => a.Id == existsAdvertisment.Id);
modifyAdvertisment.modifyDate = Utils.GetUkranianDateTimeNow();
context.SubmitChanges();
}
}
catch (Exception e)
{
WriteLog("Saving advertisment error. " + Environment.NewLine
+ "Error: " + e.Message + Environment.NewLine
+ ". Trace:" + e.StackTrace);
}
}
WriteLog("Finished. Saved advertisments in DB.");
return(savedAdvertismentsCount);
}