Exemple #1
0
    public async Task RunAsync(
        [TimerTrigger("0 */2 * * * *")] TimerInfo myTimer)
    {
        var startedAt = DateTime.UtcNow;

        _logger.LogDebug(
            "{Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos} Collector started at: {startedAt}",
            Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos, startedAt);

        var configuration = await _configurationRepository.GetAsync(
            Constants.Tables.Configuration,
            Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos) ??
                            new CollectorConfiguration(Constants.ConfigurationFunctionNames
                                                       .CollectorsYouTubeLoadNewVideos)
        {
            LastCheckedFeed = startedAt, LastItemAddedOrUpdated = DateTime.MinValue
        };

        // Check for new items
        _logger.LogDebug($"Checking playlist for videos since '{configuration.LastItemAddedOrUpdated}'",
                         configuration.LastItemAddedOrUpdated);
        var newItems = await _youTubeReader.GetAsync(configuration.LastItemAddedOrUpdated);

        // If there is nothing new, save the last checked value and exit
        if (newItems == null || newItems.Count == 0)
        {
            configuration.LastCheckedFeed = startedAt;
            await _configurationRepository.SaveAsync(configuration);

            _logger.LogDebug("No new videos found in the playlist.");
            return;
        }

        // Save the new items to SourceDataRepository
        // TODO: Handle duplicate videos?
        // GitHub Issue #6
        var savedCount      = 0;
        var eventsToPublish = new List <SourceData>();

        foreach (var item in newItems)
        {
            // shorten the url
            item.ShortenedUrl = await _urlShortener.GetShortenedUrlAsync(item.Url, _settings.BitlyShortenedDomain);

            // attempt to save the item
            try
            {
                var wasSaved = await _sourceDataRepository.SaveAsync(item);

                if (wasSaved)
                {
                    eventsToPublish.Add(item);
                    _telemetryClient.TrackEvent(Constants.Metrics.VideoAddedOrUpdated, item.ToDictionary());
                    savedCount++;
                }
                else
                {
                    _logger.LogError("Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'",
                                     item.Id, item.Url);
                }
            }
            catch (Exception e)
            {
                _logger.LogError(e,
                                 "Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'. Exception: {e.Message}",
                                 item.Id, item.Url, e);
            }
        }

        // Publish the events

        var eventsPublished = await _eventPublisher.PublishEventsAsync(_settings.TopicNewSourceDataEndpoint,
                                                                       _settings.TopicNewSourceDataKey,
                                                                       Constants.ConfigurationFunctionNames.CollectorsFeedLoadNewPosts, eventsToPublish);

        if (!eventsPublished)
        {
            _logger.LogError("Failed to publish the events for the new or updated videos.");
        }

        // Save the last checked value
        configuration.LastCheckedFeed = startedAt;
        var latestAdded   = newItems.Max(item => item.PublicationDate);
        var latestUpdated = newItems.Max(item => item.UpdatedOnDate);

        configuration.LastItemAddedOrUpdated = latestUpdated > latestAdded
            ? latestUpdated.Value.ToUniversalTime()
            : latestAdded.ToUniversalTime();

        await _configurationRepository.SaveAsync(configuration);

        // Return
        var doneMessage = $"Loaded {savedCount} of {newItems.Count} video(s).";

        _logger.LogDebug(doneMessage);
    }
Exemple #2
0
    public async Task <IActionResult> RunAsync(
        [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)]
        Domain.Models.LoadJsonFeedItemsRequest requestModel,
        HttpRequest req)
    {
        var startedAt = DateTime.UtcNow;

        _logger.LogDebug(
            $"{Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadAllVideos} Collector started at: {startedAt}",
            Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadAllVideos, startedAt);

        // Check for the from date
        var dateToCheckFrom = DateTime.MinValue;

        if (requestModel != null)
        {
            dateToCheckFrom = requestModel.CheckFrom;
        }

        _logger.LogInformation($"Getting all items from YouTube for the playlist since '{dateToCheckFrom}'.", dateToCheckFrom);
        var newItems = await _youTubeReader.GetAsync(dateToCheckFrom);

        // If there is nothing new, save the last checked value and exit
        if (newItems == null || newItems.Count == 0)
        {
            _logger.LogInformation("No videos found in the playlist.");
            return(new OkObjectResult("0 videos were found"));
        }

        // Save the new items to SourceDataRepository
        // TODO: Handle duplicate posts?
        // GitHub Issue #17
        var savedCount = 0;

        foreach (var item in newItems)
        {
            // shorten the url
            item.ShortenedUrl = await _urlShortener.GetShortenedUrlAsync(item.Url, _settings.BitlyShortenedDomain);

            // attempt to save the item
            try
            {
                var saveWasSuccessful = await _sourceDataRepository.SaveAsync(item);

                if (saveWasSuccessful)
                {
                    _telemetryClient.TrackEvent(Constants.Metrics.VideoAddedOrUpdated, item.ToDictionary());
                    savedCount++;
                }
                else
                {
                    _logger.LogError("Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'", item.Id, item.Url);
                }
            }
            catch (Exception e)
            {
                _logger.LogError(e,
                                 "Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'. Exception: {e.Message}",
                                 item.Id, item.Url, e);
            }
        }

        // Return
        var doneMessage = $"Loaded {savedCount} of {newItems.Count} videos(s).";

        _logger.LogInformation(doneMessage);
        return(new OkObjectResult(doneMessage));
    }