private async Task HandleEncodingJobEvent(EncodingJobEvent notification)
{
string jobId = notification.GetJobId();
// Lookup the uploaded video's Id by job Id
PreparedStatement lookupPrepared =
await _statementCache.NoContext.GetOrAddAsync("SELECT videoid FROM uploaded_video_jobs_by_jobid WHERE jobid = ?");
RowSet lookupRows = await _session.ExecuteAsync(lookupPrepared.Bind(jobId)).ConfigureAwait(false);
Row lookupRow = lookupRows.SingleOrDefault();
if (lookupRow == null)
{
throw new InvalidOperationException(string.Format("Could not find video for job id {0}. Were multiple jobs started for a video?", jobId));
}
var videoId = lookupRow.GetValue <Guid>("videoid");
// Log the event to C* (this should be idempotent in case of dupliacte tries since we're keyed by the job id, date, and etag in C*)
PreparedStatement preparedStatement = await _statementCache.NoContext.GetOrAddAsync(
"INSERT INTO encoding_job_notifications (videoid, status_date, etag, jobId, newstate, oldstate) VALUES (?, ?, ?, ?, ?, ?) USING TIMESTAMP ?");
string newState = notification.GetNewState();
string oldState = notification.GetOldState();
DateTimeOffset statusDate = notification.TimeStamp;
// INSERT INTO encoding_job_notifications ...
await _session.ExecuteAsync(
preparedStatement.Bind(videoId, statusDate, notification.ETag, jobId, newState, oldState, statusDate.ToMicrosecondsSinceEpoch())).ConfigureAwait(false);
// See if the job has finished and if not, just bail
if (notification.IsJobFinished() == false)
{
return;
}
// Publish the appropriate event based on whether the job was successful or not
if (notification.WasSuccessful())
{
await _bus.Publish(new UploadedVideoProcessingSucceeded
{
VideoId = videoId,
Timestamp = statusDate
}).ConfigureAwait(false);
return;
}
await _bus.Publish(new UploadedVideoProcessingFailed
{
VideoId = videoId,
Timestamp = statusDate
}).ConfigureAwait(false);
}
private async Task HandleEncodingJobEvent(EncodingJobEvent notification)
{
string jobId = notification.GetJobId();
// Lookup the uploaded video's Id by job Id
PreparedStatement lookupPrepared =
await _statementCache.NoContext.GetOrAddAsync("SELECT videoid FROM uploaded_video_jobs_by_jobid WHERE jobid = ?");
RowSet lookupRows = await _session.ExecuteAsync(lookupPrepared.Bind(jobId)).ConfigureAwait(false);
Row lookupRow = lookupRows.SingleOrDefault();
if (lookupRow == null)
throw new InvalidOperationException(string.Format("Could not find video for job id {0}. Were multiple jobs started for a video?", jobId));
var videoId = lookupRow.GetValue<Guid>("videoid");
// Log the event to C* (this should be idempotent in case of dupliacte tries since we're keyed by the job id, date, and etag in C*)
PreparedStatement preparedStatement = await _statementCache.NoContext.GetOrAddAsync(
"INSERT INTO encoding_job_notifications (videoid, status_date, etag, jobId, newstate, oldstate) VALUES (?, ?, ?, ?, ?, ?) USING TIMESTAMP ?");
string newState = notification.GetNewState();
string oldState = notification.GetOldState();
DateTimeOffset statusDate = notification.TimeStamp;
// INSERT INTO encoding_job_notifications ...
await _session.ExecuteAsync(
preparedStatement.Bind(videoId, statusDate, notification.ETag, jobId, newState, oldState, statusDate.ToMicrosecondsSinceEpoch())).ConfigureAwait(false);
// See if the job has finished and if not, just bail
if (notification.IsJobFinished() == false)
return;
// Publish the appropriate event based on whether the job was successful or not
if (notification.WasSuccessful())
{
await _bus.Publish(new UploadedVideoProcessingSucceeded
{
VideoId = videoId,
Timestamp = statusDate
}).ConfigureAwait(false);
return;
}
await _bus.Publish(new UploadedVideoProcessingFailed
{
VideoId = videoId,
Timestamp = statusDate
}).ConfigureAwait(false);
}
private async Task ExecuteImpl(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (_initialized == false)
{
await Initialize().ConfigureAwait(false);
}
bool gotSomeMessages;
do
{
// Always start by assuming we won't get any messages from the queue
gotSomeMessages = false;
// Get a batch of messages
IEnumerable <CloudQueueMessage> messages = await _queue.GetMessagesAsync(MessagesPerGet, cancellationToken).ConfigureAwait(false);
foreach (CloudQueueMessage message in messages)
{
// Check for cancellation before processing a message
cancellationToken.ThrowIfCancellationRequested();
// We obviously got some messages since we're processing one
gotSomeMessages = true;
// Try to deserialize the message to an EncodingJobEvent
EncodingJobEvent jobEvent = null;
try
{
var settings = new JsonSerializerSettings {
DateTimeZoneHandling = DateTimeZoneHandling.Utc
};
jobEvent = JsonConvert.DeserializeObject <EncodingJobEvent>(message.AsString, settings);
}
catch (Exception e)
{
Logger.Warn("Exception while deserializing event, message will be deleted.", e);
}
// If there was a problem with deserialization, just assume a poison message and delete it
if (jobEvent == null)
{
await _queue.DeleteMessageAsync(message, cancellationToken).ConfigureAwait(false);
continue;
}
// Ignore any messages that aren't for JobStateChanges
if (jobEvent.IsJobStateChangeEvent() == false)
{
await _queue.DeleteMessageAsync(message, cancellationToken).ConfigureAwait(false);
continue;
}
// Otherwise, handle the event
bool handledSuccessfully = false;
try
{
await HandleEncodingJobEvent(jobEvent).ConfigureAwait(false);
handledSuccessfully = true;
}
catch (Exception e)
{
Logger.Error("Error while handling JobStateChanged message", e);
}
// If the message was handled successfully, just delete it
if (handledSuccessfully)
{
await _queue.DeleteMessageAsync(message, cancellationToken).ConfigureAwait(false);
continue;
}
// If the message is over the number of retries, consider it a poison message, log it and delete it
if (jobEvent.RetryAttempts >= MaxRetries)
{
// Move the message to a poison queue (NOTE: because Add + Delete aren't "transactional" it is possible
// a poison message might get added more than once to the poison queue, but that's OK)
Logger.Fatal(string.Format("Giving up on message: {0}", message.AsString));
await _poisonQueue.AddMessageAsync(message, cancellationToken).ConfigureAwait(false);
await _queue.DeleteMessageAsync(message, cancellationToken).ConfigureAwait(false);
continue;
}
// Increment the retry attempts and then modify the message in place so it will be processed again
int secondsUntilRetry = (2 ^ jobEvent.RetryAttempts) * 10;
jobEvent.RetryAttempts++;
message.SetMessageContent(JsonConvert.SerializeObject(jobEvent));
await _queue.UpdateMessageAsync(message, TimeSpan.FromSeconds(secondsUntilRetry),
MessageUpdateFields.Content | MessageUpdateFields.Visibility, cancellationToken).ConfigureAwait(false);
}
// If we got some messages from the queue, keep processing until we don't get any
} while (gotSomeMessages);
// Exit method to allow a cooldown period (10s) between polling the queue whenever we run out of messages
}
