public async Task RunAsync(
[TimerTrigger("0 */2 * * * *")] TimerInfo myTimer)
{
var startedAt = DateTime.UtcNow;
_logger.LogDebug(
"{Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos} Collector started at: {startedAt}",
Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos, startedAt);
var configuration = await _configurationRepository.GetAsync(
Constants.Tables.Configuration,
Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos) ??
new CollectorConfiguration(Constants.ConfigurationFunctionNames
.CollectorsYouTubeLoadNewVideos)
{
LastCheckedFeed = startedAt, LastItemAddedOrUpdated = DateTime.MinValue
};
// Check for new items
_logger.LogDebug($"Checking playlist for videos since '{configuration.LastItemAddedOrUpdated}'",
configuration.LastItemAddedOrUpdated);
var newItems = await _youTubeReader.GetAsync(configuration.LastItemAddedOrUpdated);
// If there is nothing new, save the last checked value and exit
if (newItems == null || newItems.Count == 0)
{
configuration.LastCheckedFeed = startedAt;
await _configurationRepository.SaveAsync(configuration);
_logger.LogDebug("No new videos found in the playlist.");
return;
}
// Save the new items to SourceDataRepository
// TODO: Handle duplicate videos?
// GitHub Issue #6
var savedCount = 0;
var eventsToPublish = new List <SourceData>();
foreach (var item in newItems)
{
// shorten the url
item.ShortenedUrl = await _urlShortener.GetShortenedUrlAsync(item.Url, _settings.BitlyShortenedDomain);
// attempt to save the item
try
{
var wasSaved = await _sourceDataRepository.SaveAsync(item);
if (wasSaved)
{
eventsToPublish.Add(item);
_telemetryClient.TrackEvent(Constants.Metrics.VideoAddedOrUpdated, item.ToDictionary());
savedCount++;
}
else
{
_logger.LogError("Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'",
item.Id, item.Url);
}
}
catch (Exception e)
{
_logger.LogError(e,
"Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'. Exception: {e.Message}",
item.Id, item.Url, e);
}
}
// Publish the events
var eventsPublished = await _eventPublisher.PublishEventsAsync(_settings.TopicNewSourceDataEndpoint,
_settings.TopicNewSourceDataKey,
Constants.ConfigurationFunctionNames.CollectorsFeedLoadNewPosts, eventsToPublish);
if (!eventsPublished)
{
_logger.LogError("Failed to publish the events for the new or updated videos.");
}
// Save the last checked value
configuration.LastCheckedFeed = startedAt;
var latestAdded = newItems.Max(item => item.PublicationDate);
var latestUpdated = newItems.Max(item => item.UpdatedOnDate);
configuration.LastItemAddedOrUpdated = latestUpdated > latestAdded
? latestUpdated.Value.ToUniversalTime()
: latestAdded.ToUniversalTime();
await _configurationRepository.SaveAsync(configuration);
// Return
var doneMessage = $"Loaded {savedCount} of {newItems.Count} video(s).";
_logger.LogDebug(doneMessage);
}
public async Task RunAsync([TimerTrigger("0 */2 * * * *")] TimerInfo myTimer, ILogger log)
{
var startedAt = DateTime.UtcNow;
_logger.LogDebug(
$"{Constants.ConfigurationFunctionNames.PublishersScheduledItems} Publisher started at: {{startedAt}}",
Constants.ConfigurationFunctionNames.PublishersRandomPosts);
var configuration = await _configurationRepository.GetAsync(Constants.Tables.Configuration,
Constants.ConfigurationFunctionNames.PublishersScheduledItems
) ??
new CollectorConfiguration(Constants.ConfigurationFunctionNames
.PublishersScheduledItems)
{
LastCheckedFeed = startedAt, LastItemAddedOrUpdated = DateTime.MinValue
};
// Check for items that are due to be fired
_logger.LogDebug("Checking for scheduled items that have not been fired");
var scheduledItems =
await _scheduledItemManager.GetUpcomingScheduledItemsAsync();
// If there are no scheduled items, log it, and exit
if (scheduledItems is null || scheduledItems.Count == 0)
{
configuration.LastCheckedFeed = startedAt;
await _configurationRepository.SaveAsync(configuration);
_logger.LogDebug("No new scheduled items found");
return;
}
// Iterate through the scheduled item
foreach (var scheduledItem in scheduledItems)
{
_telemetryClient.TrackEvent(Constants.Metrics.ScheduledItemFired, scheduledItem.ToDictionary());
}
// Publish the events
var eventsPublished = await _eventPublisher.PublishEventsAsync(_settings.TopicScheduledItemFiredDataEndpoint,
_settings.TopicScheduledItemFiredDataKey,
Constants.ConfigurationFunctionNames.PublishersScheduledItems, scheduledItems);
if (!eventsPublished)
{
_logger.LogError("Failed to publish the events for some scheduled items");
}
else
{
// Mark the messages as sent
foreach (var scheduledItem in scheduledItems)
{
var wasSent = await _scheduledItemManager.SentScheduledItemAsync(scheduledItem.Id);
if (!wasSent)
{
_logger.LogWarning(
"Failed to update the sent flag for scheduled items with the id of '{scheduledItem.Id}'",
scheduledItem.Id);
}
}
}
// Save the last checked value
configuration.LastCheckedFeed = startedAt;
await _configurationRepository.SaveAsync(configuration);
_logger.LogDebug("Done publishing the events for schedule items.");
}
