示例#1
0
        public async Task CheckForDailyData(DbContextType db, LastUpdateContext lastUpdateDb)
        {
            var updateNeeded = await this.CheckIfUpdateNeeded(db, lastUpdateDb);

            if (updateNeeded)
            {
                var downloadUrl = await this.GetDownloadUrl();

                using (var httpClient = new HttpClient())
                {
                    httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(CUserAgent);

                    using (var response = await httpClient.GetAsync(downloadUrl))
                    {
                        response.EnsureSuccessStatusCode();

                        using (var stream = await response.Content.ReadAsStreamAsync())
                            using (var streamReader = new StreamReader(stream)) await ParseAndInsertNewRecords(db, streamReader);
                    }
                }

                // set last update to now
                var entity = await lastUpdateDb.LastUpdates.AsNoTracking().SingleAsync(this.LastUpdatePredicate);

                entity.Date = DateTime.UtcNow;
                lastUpdateDb.Update(entity);
                await lastUpdateDb.SaveChangesAsync();
            }
        }
        private async Task <bool> CheckIfUpdateNeeded(LastUpdateContext dbContext)
        {
            var lastUpdate = await this.GetLastUpdateAsync(dbContext);

            // default to true since we have a check interval
            return(true);
        }
示例#3
0
        protected async Task <DateTime> GetLastUpdateAsync(LastUpdateContext db)
        {
            var lastUpdate = await db.LastUpdates.SingleOrDefaultAsync(this.LastUpdatePredicate);

            if (lastUpdate == null)
            {
                db.LastUpdates.Add(new LastUpdate {
                    Id = FeedId, Date = DateTime.MinValue
                });
                await db.SaveChangesAsync();

                return(DateTime.MinValue);
            }

            return(lastUpdate.Date);
        }
        protected override async Task <bool> CheckIfUpdateNeeded(CountryContext db, LastUpdateContext lastUpdateDb)
        {
            DateTime yesterday = DateTime.Now.AddDays(-1);

            if (await db.CountriesData.FirstOrDefaultAsync(w => w.Date.Date == yesterday.Date) == null)
            {
                // if there's no data from yesterday
                var lastUpdate = await this.GetLastUpdateAsync(lastUpdateDb);

                // default to true since we have a check interval
                return(true);
            }

            _logger.LogInformation("No need to update - data already exists from yesterday.");
            return(false);
        }
示例#5
0
 protected abstract Task <bool> CheckIfUpdateNeeded(DbContextType db, LastUpdateContext lastUpdateDb);
        private async Task CheckForNews(RssNewsContext rssNewsContext, LastUpdateContext lastUpdateContext)
        {
            var updateNeeded = await this.CheckIfUpdateNeeded(lastUpdateContext);

            if (updateNeeded)
            {
                var feedUrl = string.Format(_feedUrl, _locale, _country);

                _logger.LogDebug($"Checking feed from: {feedUrl}");

                var feed = await FeedReader.ReadAsync(feedUrl);

                _logger.LogDebug($"Found {feed.Items.Count} items in feed.");

                var addCnt = 0;
                foreach (var item in feed.Items)
                {
                    // prevent duplicates if ID has changed
                    if (await rssNewsContext.News.FirstOrDefaultAsync(w => w.Id == item.Id || w.Link == item.Link) != null)
                    {
                        continue;
                    }

                    // news missing -- needs to be inserted

                    _logger.LogDebug($"Parsing news {item.Title}::{item.PublishingDateString}");

                    if (item.PublishingDate.HasValue && item.PublishingDate < DateTime.UtcNow.AddDays(-_settings.RetentionInDays))
                    {
                        _logger.LogDebug("News too old, skipping.");
                        continue;
                    }

                    var mrssItem = item.SpecificItem as MediaRssFeedItem;

                    byte[] img    = null;
                    var    endUrl = await this.CheckForRedirectsAsync(item.Link);

                    if (endUrl != null)
                    {
                        img = await this.GetThumbnail(endUrl);
                    }

                    rssNewsContext.News.Add(new Model.RssNews
                    {
                        Id         = item.Id,
                        FeedId     = FeedId,
                        Date       = item.PublishingDate.HasValue ? item.PublishingDate.Value.ToUniversalTime() : DateTime.UtcNow,
                        Title      = item.Title,
                        Link       = item.Link,
                        EndUrl     = endUrl,
                        ImageData  = img,
                        SourceName = mrssItem?.Source?.Value,
                        SourceUrl  = mrssItem?.Source?.Url,
                    });
                    addCnt++;

                    // break after adding some news
                    if (addCnt == _settings.MaxNewsInsertAtOnce)
                    {
                        break;
                    }
                }

                if (addCnt > 0)
                {
                    rssNewsContext.SaveChanges();
                }
                _logger.LogInformation($"Added {addCnt} news in the database");

                // set last update to now
                var entity = await lastUpdateContext.LastUpdates.AsNoTracking().SingleAsync(x => x.Id == FeedId);

                entity.Date = DateTime.UtcNow;
                lastUpdateContext.Update(entity);
                await lastUpdateContext.SaveChangesAsync();
            }
        }