public async Task CheckForDailyData(DbContextType db, LastUpdateContext lastUpdateDb)
{
var updateNeeded = await this.CheckIfUpdateNeeded(db, lastUpdateDb);
if (updateNeeded)
{
var downloadUrl = await this.GetDownloadUrl();
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(CUserAgent);
using (var response = await httpClient.GetAsync(downloadUrl))
{
response.EnsureSuccessStatusCode();
using (var stream = await response.Content.ReadAsStreamAsync())
using (var streamReader = new StreamReader(stream)) await ParseAndInsertNewRecords(db, streamReader);
}
}
// set last update to now
var entity = await lastUpdateDb.LastUpdates.AsNoTracking().SingleAsync(this.LastUpdatePredicate);
entity.Date = DateTime.UtcNow;
lastUpdateDb.Update(entity);
await lastUpdateDb.SaveChangesAsync();
}
}
private async Task <bool> CheckIfUpdateNeeded(LastUpdateContext dbContext)
{
var lastUpdate = await this.GetLastUpdateAsync(dbContext);
// default to true since we have a check interval
return(true);
}
protected async Task <DateTime> GetLastUpdateAsync(LastUpdateContext db)
{
var lastUpdate = await db.LastUpdates.SingleOrDefaultAsync(this.LastUpdatePredicate);
if (lastUpdate == null)
{
db.LastUpdates.Add(new LastUpdate {
Id = FeedId, Date = DateTime.MinValue
});
await db.SaveChangesAsync();
return(DateTime.MinValue);
}
return(lastUpdate.Date);
}
protected override async Task <bool> CheckIfUpdateNeeded(CountryContext db, LastUpdateContext lastUpdateDb)
{
DateTime yesterday = DateTime.Now.AddDays(-1);
if (await db.CountriesData.FirstOrDefaultAsync(w => w.Date.Date == yesterday.Date) == null)
{
// if there's no data from yesterday
var lastUpdate = await this.GetLastUpdateAsync(lastUpdateDb);
// default to true since we have a check interval
return(true);
}
_logger.LogInformation("No need to update - data already exists from yesterday.");
return(false);
}
protected abstract Task <bool> CheckIfUpdateNeeded(DbContextType db, LastUpdateContext lastUpdateDb);
private async Task CheckForNews(RssNewsContext rssNewsContext, LastUpdateContext lastUpdateContext)
{
var updateNeeded = await this.CheckIfUpdateNeeded(lastUpdateContext);
if (updateNeeded)
{
var feedUrl = string.Format(_feedUrl, _locale, _country);
_logger.LogDebug($"Checking feed from: {feedUrl}");
var feed = await FeedReader.ReadAsync(feedUrl);
_logger.LogDebug($"Found {feed.Items.Count} items in feed.");
var addCnt = 0;
foreach (var item in feed.Items)
{
// prevent duplicates if ID has changed
if (await rssNewsContext.News.FirstOrDefaultAsync(w => w.Id == item.Id || w.Link == item.Link) != null)
{
continue;
}
// news missing -- needs to be inserted
_logger.LogDebug($"Parsing news {item.Title}::{item.PublishingDateString}");
if (item.PublishingDate.HasValue && item.PublishingDate < DateTime.UtcNow.AddDays(-_settings.RetentionInDays))
{
_logger.LogDebug("News too old, skipping.");
continue;
}
var mrssItem = item.SpecificItem as MediaRssFeedItem;
byte[] img = null;
var endUrl = await this.CheckForRedirectsAsync(item.Link);
if (endUrl != null)
{
img = await this.GetThumbnail(endUrl);
}
rssNewsContext.News.Add(new Model.RssNews
{
Id = item.Id,
FeedId = FeedId,
Date = item.PublishingDate.HasValue ? item.PublishingDate.Value.ToUniversalTime() : DateTime.UtcNow,
Title = item.Title,
Link = item.Link,
EndUrl = endUrl,
ImageData = img,
SourceName = mrssItem?.Source?.Value,
SourceUrl = mrssItem?.Source?.Url,
});
addCnt++;
// break after adding some news
if (addCnt == _settings.MaxNewsInsertAtOnce)
{
break;
}
}
if (addCnt > 0)
{
rssNewsContext.SaveChanges();
}
_logger.LogInformation($"Added {addCnt} news in the database");
// set last update to now
var entity = await lastUpdateContext.LastUpdates.AsNoTracking().SingleAsync(x => x.Id == FeedId);
entity.Date = DateTime.UtcNow;
lastUpdateContext.Update(entity);
await lastUpdateContext.SaveChangesAsync();
}
}