public async Task RunAsync(
[TimerTrigger("0 */2 * * * *")] TimerInfo myTimer)
{
var startedAt = DateTime.UtcNow;
_logger.LogDebug(
"{Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos} Collector started at: {startedAt}",
Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos, startedAt);
var configuration = await _configurationRepository.GetAsync(
Constants.Tables.Configuration,
Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadNewVideos) ??
new CollectorConfiguration(Constants.ConfigurationFunctionNames
.CollectorsYouTubeLoadNewVideos)
{
LastCheckedFeed = startedAt, LastItemAddedOrUpdated = DateTime.MinValue
};
// Check for new items
_logger.LogDebug($"Checking playlist for videos since '{configuration.LastItemAddedOrUpdated}'",
configuration.LastItemAddedOrUpdated);
var newItems = await _youTubeReader.GetAsync(configuration.LastItemAddedOrUpdated);
// If there is nothing new, save the last checked value and exit
if (newItems == null || newItems.Count == 0)
{
configuration.LastCheckedFeed = startedAt;
await _configurationRepository.SaveAsync(configuration);
_logger.LogDebug("No new videos found in the playlist.");
return;
}
// Save the new items to SourceDataRepository
// TODO: Handle duplicate videos?
// GitHub Issue #6
var savedCount = 0;
var eventsToPublish = new List <SourceData>();
foreach (var item in newItems)
{
// shorten the url
item.ShortenedUrl = await _urlShortener.GetShortenedUrlAsync(item.Url, _settings.BitlyShortenedDomain);
// attempt to save the item
try
{
var wasSaved = await _sourceDataRepository.SaveAsync(item);
if (wasSaved)
{
eventsToPublish.Add(item);
_telemetryClient.TrackEvent(Constants.Metrics.VideoAddedOrUpdated, item.ToDictionary());
savedCount++;
}
else
{
_logger.LogError("Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'",
item.Id, item.Url);
}
}
catch (Exception e)
{
_logger.LogError(e,
"Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'. Exception: {e.Message}",
item.Id, item.Url, e);
}
}
// Publish the events
var eventsPublished = await _eventPublisher.PublishEventsAsync(_settings.TopicNewSourceDataEndpoint,
_settings.TopicNewSourceDataKey,
Constants.ConfigurationFunctionNames.CollectorsFeedLoadNewPosts, eventsToPublish);
if (!eventsPublished)
{
_logger.LogError("Failed to publish the events for the new or updated videos.");
}
// Save the last checked value
configuration.LastCheckedFeed = startedAt;
var latestAdded = newItems.Max(item => item.PublicationDate);
var latestUpdated = newItems.Max(item => item.UpdatedOnDate);
configuration.LastItemAddedOrUpdated = latestUpdated > latestAdded
? latestUpdated.Value.ToUniversalTime()
: latestAdded.ToUniversalTime();
await _configurationRepository.SaveAsync(configuration);
// Return
var doneMessage = $"Loaded {savedCount} of {newItems.Count} video(s).";
_logger.LogDebug(doneMessage);
}
public async Task <IActionResult> RunAsync(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)]
Domain.Models.LoadJsonFeedItemsRequest requestModel,
HttpRequest req)
{
var startedAt = DateTime.UtcNow;
_logger.LogDebug(
$"{Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadAllVideos} Collector started at: {startedAt}",
Constants.ConfigurationFunctionNames.CollectorsYouTubeLoadAllVideos, startedAt);
// Check for the from date
var dateToCheckFrom = DateTime.MinValue;
if (requestModel != null)
{
dateToCheckFrom = requestModel.CheckFrom;
}
_logger.LogInformation($"Getting all items from YouTube for the playlist since '{dateToCheckFrom}'.", dateToCheckFrom);
var newItems = await _youTubeReader.GetAsync(dateToCheckFrom);
// If there is nothing new, save the last checked value and exit
if (newItems == null || newItems.Count == 0)
{
_logger.LogInformation("No videos found in the playlist.");
return(new OkObjectResult("0 videos were found"));
}
// Save the new items to SourceDataRepository
// TODO: Handle duplicate posts?
// GitHub Issue #17
var savedCount = 0;
foreach (var item in newItems)
{
// shorten the url
item.ShortenedUrl = await _urlShortener.GetShortenedUrlAsync(item.Url, _settings.BitlyShortenedDomain);
// attempt to save the item
try
{
var saveWasSuccessful = await _sourceDataRepository.SaveAsync(item);
if (saveWasSuccessful)
{
_telemetryClient.TrackEvent(Constants.Metrics.VideoAddedOrUpdated, item.ToDictionary());
savedCount++;
}
else
{
_logger.LogError("Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'", item.Id, item.Url);
}
}
catch (Exception e)
{
_logger.LogError(e,
"Failed to save the video with the id of: '{item.Id}' Url:'{item.Url}'. Exception: {e.Message}",
item.Id, item.Url, e);
}
}
// Return
var doneMessage = $"Loaded {savedCount} of {newItems.Count} videos(s).";
_logger.LogInformation(doneMessage);
return(new OkObjectResult(doneMessage));
}
