public async Task UpdateFromSitemap(string postSiteMapURL, IXMLSiteMapLoader siteMapLoader, ILogger log)
{
//Здесь мы не даем лимиты, по идее каждый раз сканировать всю карту сайта не нужно, а лишь последние 10 постов
URLInfo[] urls = await siteMapLoader.Load(postSiteMapURL);
if (urls == null || urls.Length == 0)
{
throw new ApplicationException("urls null or empty");
}
log.Log("Found {0} urls from sitemap...", urls.Length);
//по каждому урлу надо уточнить, есть ли он в базе, если нет - добавляем
int added = 0;
foreach (var urlInfo in urls)
{
//поиск по URL - это достаточно надежный "ключ" уникальности записи
string url = urlInfo.URL;
var findResult = await _SitePages.FindAsync(x => x.URL == url);
var foundRecord = findResult.FirstOrDefault();
if (foundRecord != null)
{
continue;
}
SitePage sitePage = new SitePage
{
LastModified = urlInfo.LastModified,
Title = urlInfo.Title,
URL = url,
SpecialDay = SpecialDays.DetectSpecialDay(url)
};
await _SitePages.InsertOneAsync(sitePage);
added++;
log.Log("Added new url: {0}", url);
} //foreach
log.Log("Done updating from sitemap, added {0}", added);
}
private void GetMonth(DateTime day)
{
// add holidays
var holidays = new HolidayCalculator(day, Utils.ReadConfig("Holidays", "Path"));
foreach (HolidayCalculator.Holiday h in holidays.OrderedHolidays)
{
if (h.Date.Month == day.Month)
{
Holidays.Add(new Holiday(h.Name, h.Date));
}
// and days before holidays if theye're not weekends or holidays
var yesterday = h.Date.AddDays(-1);
if (yesterday.Month == day.Month &&
Holidays.FirstOrDefault(hl => hl.Date == yesterday) == default(Holiday) &&
yesterday.DayOfWeek != DayOfWeek.Saturday &&
yesterday.DayOfWeek != DayOfWeek.Sunday)
{
SpecialDays.Add(new SpecialDay("Day before " + h.Name, yesterday.Date));
}
}
// if the last day in month is day before holiday
var firstInNextMonth = Last.AddDays(1);
var hol = Holidays.FirstOrDefault(h => h.Date == firstInNextMonth);
if (hol != null &&
Last.Date.DayOfWeek != DayOfWeek.Saturday &&
Last.Date.DayOfWeek != DayOfWeek.Sunday)
{
SpecialDays.Add(new SpecialDay("Day before " + hol.Name, Last.Date));
}
// get all fridays, if they're not holidays
var startDate = First;
while (startDate.DayOfWeek != DayOfWeek.Friday)
{
startDate = startDate.AddDays(1);
}
while (startDate < Last)
{
if (Holidays.FirstOrDefault(h => h.Date == startDate) == default(Holiday))
{
SpecialDays.Add(new SpecialDay("Friday", startDate));
}
startDate = startDate.AddDays(7);
}
// get all weekends, if they're not holidays
startDate = First;
while (startDate.DayOfWeek != DayOfWeek.Saturday)
{
startDate = startDate.AddDays(1);
}
while (startDate < Last)
{
if (Holidays.FirstOrDefault(h => h.Date == startDate) == default(Holiday))
{
NonWorkingDays.Add(new NonWorkingDay(startDate));
}
startDate = startDate.AddDays(1);
if (Holidays.FirstOrDefault(h => h.Date == startDate) == default(Holiday))
{
NonWorkingDays.Add(new NonWorkingDay(startDate));
}
startDate = startDate.AddDays(6);
}
}