public Mp4boxJob GetNextDashJob()
{
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var connection = Helper.GetConnection())
{
connection.Open();
var data = connection.Query <Mp4boxJob>(
"SELECT TOP 1 JobCorrelationId, Arguments FROM Mp4boxJobs WHERE State = @State ORDER BY Needed ASC, Id ASC;",
new { State = TranscodingJobState.Queued })
.SingleOrDefault();
if (data == null)
{
return(null);
}
var rowsUpdated =
connection.Execute("UPDATE Mp4boxJobs SET State = @State WHERE JobCorrelationId = @Id;",
new { State = TranscodingJobState.InProgress, Id = data.JobCorrelationId });
if (rowsUpdated != 1)
{
return(null);
}
scope.Complete();
return(new Mp4boxJob
{
JobCorrelationId = data.JobCorrelationId,
Arguments = data.Arguments
});
}
}
}
public bool PauseJob(Guid jobId)
{
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var conn = Helper.GetConnection())
{
JobType jobType = GetJobType(jobId, conn);
string sql = "UPDATE FfmpegJobs SET JobState = @PausedState WHERE JobCorrelationId = @Id AND JobState = @QueuedState;";
switch (jobType)
{
case JobType.Audio:
case JobType.Mux:
sql += "UPDATE Tasks SET Tasks.TaskState = @PausedState FROM FfmpegTasks Tasks INNER JOIN FfmpegJobs Jobs ON Jobs.id = Tasks.FfmpegJobs_Id WHERE Jobs.JobCorrelationId = @Id AND Tasks.TaskState = @QueuedState;";
break;
case JobType.Video:
case JobType.VideoMp4box:
case JobType.VideoMerge:
throw new NotImplementedException();
case JobType.Unknown:
default:
throw new ArgumentOutOfRangeException();
}
int updatedRows = conn.Execute(sql,
new { Id = jobId, PausedState = TranscodingJobState.Paused, QueuedState = TranscodingJobState.Queued });
scope.Complete();
return(updatedRows > 0);
}
}
}
public int PruneInactiveClients(TimeSpan maxAge)
{
using (var scope = TransactionUtils.CreateTransactionScope(IsolationLevel.Serializable))
{
using (var connection = Helper.GetConnection())
{
var res = connection.Execute("DELETE FROM Clients WHERE Clients.LastHeartbeat < @MaxAge",
new { MaxAge = DateTimeOffset.Now - maxAge });
scope.Complete();
return(res);
}
}
}
public MergeJob GetNextMergeJob()
{
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var connection = Helper.GetConnection())
{
connection.Open();
int timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]);
DateTime timeout = DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(timeoutSeconds));
var data = connection.Query <dynamic>(
"SELECT TOP 1 Id, Arguments, JobCorrelationId FROM FfmpegMergeJobs WHERE State = @QueuedState OR (State = @InProgressState AND HeartBeat < @Heartbeat) ORDER BY Needed ASC, Id ASC;",
new
{
QueuedState = TranscodingJobState.Queued,
InProgressState = TranscodingJobState.InProgress,
Heartbeat = timeout
})
.SingleOrDefault();
if (data == null)
{
return(null);
}
var rowsUpdated = connection.Execute(
"UPDATE FfmpegMergeJobs SET State = @State, HeartBeat = @Heartbeat WHERE Id = @Id;",
new { State = TranscodingJobState.InProgress, Heartbeat = DateTime.UtcNow, Id = data.Id });
if (rowsUpdated == 0)
{
throw new Exception("Failed to mark row as taken");
}
scope.Complete();
return(new MergeJob
{
Id = Convert.ToInt32(data.Id),
Arguments = new string[] { data.Arguments },
JobCorrelationId = data.JobCorrelationId
});
}
}
}
public bool CancelJob(Guid jobId)
{
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var conn = Helper.GetConnection())
{
string sql = "UPDATE FfmpegJobs SET JobState = @CanceledState WHERE JobCorrelationId = @Id AND jobState <> @DoneState AND jobState <> @FailedState;";
sql += "UPDATE Tasks SET Tasks.TaskState = @CanceledState FROM FfmpegTasks Tasks INNER JOIN FfmpegJobs Jobs ON Jobs.id = Tasks.FfmpegJobs_Id " +
"WHERE Jobs.JobCorrelationId = @Id AND Tasks.TaskState <> @DoneState AND Tasks.TaskState <> @FailedState;";
int updatedRows = conn.Execute(sql,
new { Id = jobId, CanceledState = TranscodingJobState.Canceled, DoneState = TranscodingJobState.Done, FailedState = TranscodingJobState.Failed });
scope.Complete();
return(updatedRows > 0);
}
}
}
public bool DeleteJob(Guid jobId)
{
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var connection = Helper.GetConnection())
{
connection.Open();
int rowsDeleted = -1;
JobType jobType = GetJobType(jobId, connection);
switch (jobType)
{
case JobType.Audio:
case JobType.Mux:
rowsDeleted = DeleteMuxJob(jobId, connection);
break;
case JobType.Video:
rowsDeleted = DeleteVideoJob(jobId, connection);
break;
case JobType.VideoMp4box:
case JobType.VideoMerge:
throw new NotImplementedException();
case JobType.Unknown:
return(false);
default:
throw new InvalidOperationException();
}
scope.Complete();
return(rowsDeleted > 0);
}
}
}
private static void UpdateVideoJob(BaseJob job, IDbConnection connection)
{
JobRequest jobRequest = connection.Query <JobRequest>(
"SELECT JobCorrelationId, VideoSourceFilename, AudioSourceFilename, DestinationFilename, Needed, EnableDash, EnablePsnr FROM FfmpegVideoRequest WHERE JobCorrelationId = @Id",
new { Id = job.JobCorrelationId })
.SingleOrDefault();
if (jobRequest == null)
{
throw new ArgumentException($@"Job with correlation id {job.JobCorrelationId} not found");
}
jobRequest.Targets = connection.Query <DestinationFormat>(
"SELECT JobCorrelationId, Width, Height, VideoBitrate, AudioBitrate FROM FfmpegVideoRequestTargets WHERE JobCorrelationId = @Id;",
new { Id = job.JobCorrelationId })
.ToArray();
using (var scope = TransactionUtils.CreateTransactionScope())
{
Type jobType = job.GetType();
TranscodingJobState jobState = job.Failed
? TranscodingJobState.Failed
: job.Done
? TranscodingJobState.Done
: TranscodingJobState.InProgress;
if (jobType == typeof(VideoTranscodingJob))
{
int updatedRows = connection.Execute(
"UPDATE FfmpegVideoJobs SET Progress = @Progress, Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE Id = @Id;",
new
{
Id = job.Id,
Progress = job.Progress.TotalSeconds,
Heartbeat = DateTimeOffset.UtcNow.UtcDateTime,
State = jobState,
MachineName = job.MachineName,
});
if (updatedRows != 1)
{
throw new Exception($"Failed to update progress for job id {job.Id}");
}
if (jobRequest.EnablePsnr)
{
foreach (FfmpegPart chunk in ((VideoTranscodingJob)job).Chunks)
{
connection.Execute(
"UPDATE FfmpegVideoParts SET PSNR = @Psnr WHERE Id = @Id;",
new { Id = chunk.Id, Psnr = chunk.Psnr });
}
}
}
else if (jobType == typeof(MergeJob))
{
int updatedRows = connection.Execute(
"UPDATE FfmpegMergeJobs SET Progress = @Progress, Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE Id = @Id;",
new
{
Id = job.Id,
Progress = job.Progress.TotalSeconds,
Heartbeat = DateTimeOffset.UtcNow.UtcDateTime,
State = jobState,
MachineName = job.MachineName
});
if (updatedRows != 1)
{
throw new Exception($"Failed to update progress for job id {job.Id}");
}
var states = connection.Query <string>(
"SELECT State FROM FfmpegMergeJobs WHERE JobCorrelationId = @Id;",
new { Id = job.JobCorrelationId })
.Select(value => Enum.Parse(typeof(TranscodingJobState), value))
.Cast <TranscodingJobState>();
if (states.All(x => x == TranscodingJobState.Done))
{
string tempFolder = String.Concat(Path.GetDirectoryName(jobRequest.DestinationFilename),
Path.DirectorySeparatorChar, jobRequest.JobCorrelationId.ToString("N"));
Directory.Delete(tempFolder, true);
}
}
else if (jobType == typeof(Mp4boxJob))
{
int updatedRows = connection.Execute(
"UPDATE Mp4boxJobs SET Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE JobCorrelationId = @Id;",
new
{
Id = job.JobCorrelationId,
Progress = job.Progress.TotalSeconds,
Heartbeat = DateTimeOffset.UtcNow.UtcDateTime,
State = jobState,
MachineName = job.MachineName
});
if (updatedRows != 1)
{
throw new Exception($"Failed to update progress for job id {job.Id}");
}
}
scope.Complete();
}
using (var scope = TransactionUtils.CreateTransactionScope())
{
ICollection <TranscodingJobState> totalJobs = connection.Query <TranscodingJobState>(
"SELECT State FROM FfmpegVideoJobs WHERE JobCorrelationId = @Id;",
new { Id = jobRequest.JobCorrelationId })
.ToList();
if (totalJobs.Any(x => x != TranscodingJobState.Done))
{
// Not all transcoding jobs are finished
return;
}
totalJobs = connection.Query <TranscodingJobState>(
"SELECT State FROM FfmpegMergeJobs WHERE JobCorrelationId = @Id;",
new { Id = jobRequest.JobCorrelationId })
.ToList();
if (totalJobs.Any(x => x != TranscodingJobState.Done))
{
// Not all merge jobs are finished
return;
}
string destinationFilename = jobRequest.DestinationFilename;
string fileNameWithoutExtension =
Path.GetFileNameWithoutExtension(destinationFilename);
string fileExtension = Path.GetExtension(destinationFilename);
string outputFolder = Path.GetDirectoryName(destinationFilename);
if (totalJobs.Count == 0)
{
QueueMergeJob(job, connection, outputFolder, fileNameWithoutExtension, fileExtension, jobRequest);
}
else if (jobRequest.EnableDash)
{
QueueMpegDashMergeJob(job, destinationFilename, connection, jobRequest, fileNameWithoutExtension,
outputFolder, fileExtension);
}
scope.Complete();
}
}
public FFmpegTaskDto GetNextJob(string machineName)
{
if (string.IsNullOrWhiteSpace(machineName))
{
throw new ArgumentNullException(nameof(machineName));
}
InsertClientHeartbeat(machineName);
int timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]);
var now = DateTimeOffset.UtcNow;
DateTimeOffset timeout = now.Subtract(TimeSpan.FromSeconds(timeoutSeconds));
using (var connection = Helper.GetConnection())
{
connection.Open();
IEnumerable <FFmpegJobDto> existingJobs = GetActiveJobs(machineName, connection);
if (existingJobs.Any(x => x.Type == JobType.HardSubtitles))
{
return(null);
}
}
do
{
using (var scope = TransactionUtils.CreateTransactionScope(IsolationLevel.Serializable))
{
using (var connection = Helper.GetConnection())
{
try
{
var data = new
{
QueuedState = TranscodingJobState.Queued,
InProgressState = TranscodingJobState.InProgress,
Timeout = timeout,
Timestamp = now
};
connection.Open();
var task = connection.QuerySingleOrDefault <FFmpegTaskDto>("sp_GetNextTask", data, commandType: CommandType.StoredProcedure);
if (task == null)
{
return(null);
}
// Safety check to ensure that the data is being returned correctly in the SQL query
if (task.Id < 0 || task.FfmpegJobsId < 0 || string.IsNullOrWhiteSpace(task.Arguments))
{
throw new InvalidOperationException("One or more parameters were not set by SQL query.");
}
scope.Complete();
return(task);
}
catch (SqlException e)
{
// Retry in case of deadlocks
if (e.Number == 1205)
{
continue;
}
throw;
}
}
}
} while (true);
}
public VideoTranscodingJob GetNextTranscodingJob()
{
int timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]);
DateTime timeout =
DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(timeoutSeconds));
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var connection = Helper.GetConnection())
{
connection.Open();
var data = connection.Query <dynamic>(
"SELECT TOP 1 Id, Arguments, JobCorrelationId FROM FfmpegVideoJobs WHERE State = @QueuedState OR (State = @InProgressState AND HeartBeat < @Heartbeat) ORDER BY Needed ASC, Id ASC;",
new
{
QueuedState = TranscodingJobState.Queued,
InProgressState = TranscodingJobState.InProgress,
Heartbeat = timeout
})
.SingleOrDefault();
if (data == null)
{
return(null);
}
var parts =
connection.Query <dynamic>(
"SELECT Id, JobCorrelationId, Filename, Number, Target, PSNR FROM FfmpegVideoParts WHERE FfmpegVideoJobs_Id = @JobId;",
new { JobId = data.Id });
var rowsUpdated =
connection.Execute(
"UPDATE FfmpegVideoJobs SET State = @State, HeartBeat = @Heartbeat, Started = @Heartbeat WHERE Id = @Id;",
new { State = TranscodingJobState.InProgress, Heartbeat = DateTime.UtcNow, Id = data.Id });
if (rowsUpdated == 0)
{
throw new Exception("Failed to mark row as taken");
}
scope.Complete();
var job = new VideoTranscodingJob
{
Id = Convert.ToInt32(data.Id),
Arguments = data.Arguments.Split('|'),
JobCorrelationId = data.JobCorrelationId,
Chunks = parts.Select(x => new FfmpegPart
{
Id = x.Id,
JobCorrelationId = x.JobCorrelationId,
Psnr = x.PSNR,
Target = x.Target,
Number = x.Number,
SourceFilename = x.SourceFilename,
Filename = x.Filename,
}).ToList()
};
return(job);
}
}
}
public Guid Add(MuxJobRequest request, ICollection <FFmpegJob> jobs)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (jobs == null)
{
throw new ArgumentNullException(nameof(jobs));
}
Guid jobCorrelationId = Guid.NewGuid();
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var connection = new SqlConnection(_connectionString))
{
connection.Execute(
"INSERT INTO FfmpegMuxRequest (JobCorrelationId, VideoSourceFilename, AudioSourceFilename, DestinationFilename, OutputFolder, Needed, Created) VALUES(@JobCorrelationId, @VideoSourceFilename, @AudioSourceFilename, @DestinationFilename, @OutputFolder, @Needed, @Created);",
new
{
JobCorrelationId = jobCorrelationId,
request.VideoSourceFilename,
request.AudioSourceFilename,
request.Needed,
request.DestinationFilename,
request.OutputFolder,
Created = DateTime.UtcNow
});
foreach (MuxJob job in jobs.Select(x => x as MuxJob))
{
var jobId = connection.ExecuteScalar <int>(
"INSERT INTO FfmpegJobs (JobCorrelationId, Created, Needed, JobState, JobType) VALUES(@JobCorrelationId, @Created, @Needed, @State, @JobType);SELECT @@IDENTITY;",
new
{
JobCorrelationId = jobCorrelationId,
Created = DateTimeOffset.UtcNow,
job.Needed,
State = job.State,
JobType = JobType.Mux
});
connection.Execute(
"INSERT INTO FfmpegTasks (FfmpegJobs_id, Arguments, TaskState, DestinationFilename, DestinationDurationSeconds, VerifyOutput) VALUES(@FfmpegJobsId, @Arguments, @QueuedState, @DestinationFilename, @DestinationDurationSeconds, @VerifyOutput);",
new
{
FfmpegJobsId = jobId,
job.Arguments,
QueuedState = TranscodingJobState.Queued,
job.DestinationFilename,
job.DestinationDurationSeconds,
VerifyOutput = false
});
}
}
scope.Complete();
return(jobCorrelationId);
}
}
public Guid Add(AudioJobRequest request, ICollection <AudioTranscodingJob> jobs)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (jobs == null)
{
throw new ArgumentNullException(nameof(jobs));
}
if (jobs.Count == 0)
{
throw new ArgumentException("Jobs parameter must contain at least 1 job", nameof(jobs));
}
Guid jobCorrelationId = Guid.NewGuid();
using (var scope = TransactionUtils.CreateTransactionScope())
{
using (var connection = _helper.GetConnection())
{
connection.Execute(
"INSERT INTO FfmpegAudioRequest (JobCorrelationId, SourceFilename, DestinationFilename, OutputFolder, Needed, Created) VALUES(@JobCorrelationId, @SourceFilename, @DestinationFilename, @OutputFolder, @Needed, @Created);",
new
{
JobCorrelationId = jobCorrelationId,
SourceFilename = string.Join(" ", request.SourceFilenames),
request.DestinationFilename,
request.Needed,
request.OutputFolder,
Created = DateTime.UtcNow
});
foreach (AudioDestinationFormat target in request.Targets)
{
connection.Execute(
"INSERT INTO FfmpegAudioRequestTargets (JobCorrelationId, Codec, Format, Bitrate) VALUES(@JobCorrelationId, @Codec, @Format, @Bitrate);",
new
{
jobCorrelationId,
Codec = target.AudioCodec.ToString(),
Format = target.Format.ToString(),
target.Bitrate
});
}
var jobId = connection.ExecuteScalar <int>(
"INSERT INTO FfmpegJobs (JobCorrelationId, Created, Needed, JobState, JobType) VALUES(@JobCorrelationId, @Created, @Needed, @JobState, @JobType);SELECT @@IDENTITY;",
new
{
JobCorrelationId = jobCorrelationId,
Created = DateTimeOffset.UtcNow,
request.Needed,
JobState = TranscodingJobState.Queued,
JobType = JobType.Audio
});
foreach (AudioTranscodingJob transcodingJob in jobs)
{
connection.Execute(
"INSERT INTO FfmpegTasks (FfmpegJobs_id, Arguments, TaskState, DestinationFilename, DestinationDurationSeconds, VerifyOutput) VALUES(@FfmpegJobsId, @Arguments, @TaskState, @DestinationFilename, @DestinationDurationSeconds, @VerifyOutput);",
new
{
FfmpegJobsId = jobId,
transcodingJob.Arguments,
TaskState = TranscodingJobState.Queued,
transcodingJob.DestinationFilename,
transcodingJob.DestinationDurationSeconds,
VerifyOutput = true
});
}
}
scope.Complete();
return(jobCorrelationId);
}
}