public Mp4boxJob GetNextDashJob() { using (var scope = TransactionUtils.CreateTransactionScope()) { using (var connection = Helper.GetConnection()) { connection.Open(); var data = connection.Query <Mp4boxJob>( "SELECT TOP 1 JobCorrelationId, Arguments FROM Mp4boxJobs WHERE State = @State ORDER BY Needed ASC, Id ASC;", new { State = TranscodingJobState.Queued }) .SingleOrDefault(); if (data == null) { return(null); } var rowsUpdated = connection.Execute("UPDATE Mp4boxJobs SET State = @State WHERE JobCorrelationId = @Id;", new { State = TranscodingJobState.InProgress, Id = data.JobCorrelationId }); if (rowsUpdated != 1) { return(null); } scope.Complete(); return(new Mp4boxJob { JobCorrelationId = data.JobCorrelationId, Arguments = data.Arguments }); } } }
public bool PauseJob(Guid jobId) { using (var scope = TransactionUtils.CreateTransactionScope()) { using (var conn = Helper.GetConnection()) { JobType jobType = GetJobType(jobId, conn); string sql = "UPDATE FfmpegJobs SET JobState = @PausedState WHERE JobCorrelationId = @Id AND JobState = @QueuedState;"; switch (jobType) { case JobType.Audio: case JobType.Mux: sql += "UPDATE Tasks SET Tasks.TaskState = @PausedState FROM FfmpegTasks Tasks INNER JOIN FfmpegJobs Jobs ON Jobs.id = Tasks.FfmpegJobs_Id WHERE Jobs.JobCorrelationId = @Id AND Tasks.TaskState = @QueuedState;"; break; case JobType.Video: case JobType.VideoMp4box: case JobType.VideoMerge: throw new NotImplementedException(); case JobType.Unknown: default: throw new ArgumentOutOfRangeException(); } int updatedRows = conn.Execute(sql, new { Id = jobId, PausedState = TranscodingJobState.Paused, QueuedState = TranscodingJobState.Queued }); scope.Complete(); return(updatedRows > 0); } } }
public int PruneInactiveClients(TimeSpan maxAge) { using (var scope = TransactionUtils.CreateTransactionScope(IsolationLevel.Serializable)) { using (var connection = Helper.GetConnection()) { var res = connection.Execute("DELETE FROM Clients WHERE Clients.LastHeartbeat < @MaxAge", new { MaxAge = DateTimeOffset.Now - maxAge }); scope.Complete(); return(res); } } }
public MergeJob GetNextMergeJob() { using (var scope = TransactionUtils.CreateTransactionScope()) { using (var connection = Helper.GetConnection()) { connection.Open(); int timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]); DateTime timeout = DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(timeoutSeconds)); var data = connection.Query <dynamic>( "SELECT TOP 1 Id, Arguments, JobCorrelationId FROM FfmpegMergeJobs WHERE State = @QueuedState OR (State = @InProgressState AND HeartBeat < @Heartbeat) ORDER BY Needed ASC, Id ASC;", new { QueuedState = TranscodingJobState.Queued, InProgressState = TranscodingJobState.InProgress, Heartbeat = timeout }) .SingleOrDefault(); if (data == null) { return(null); } var rowsUpdated = connection.Execute( "UPDATE FfmpegMergeJobs SET State = @State, HeartBeat = @Heartbeat WHERE Id = @Id;", new { State = TranscodingJobState.InProgress, Heartbeat = DateTime.UtcNow, Id = data.Id }); if (rowsUpdated == 0) { throw new Exception("Failed to mark row as taken"); } scope.Complete(); return(new MergeJob { Id = Convert.ToInt32(data.Id), Arguments = new string[] { data.Arguments }, JobCorrelationId = data.JobCorrelationId }); } } }
public bool CancelJob(Guid jobId) { using (var scope = TransactionUtils.CreateTransactionScope()) { using (var conn = Helper.GetConnection()) { string sql = "UPDATE FfmpegJobs SET JobState = @CanceledState WHERE JobCorrelationId = @Id AND jobState <> @DoneState AND jobState <> @FailedState;"; sql += "UPDATE Tasks SET Tasks.TaskState = @CanceledState FROM FfmpegTasks Tasks INNER JOIN FfmpegJobs Jobs ON Jobs.id = Tasks.FfmpegJobs_Id " + "WHERE Jobs.JobCorrelationId = @Id AND Tasks.TaskState <> @DoneState AND Tasks.TaskState <> @FailedState;"; int updatedRows = conn.Execute(sql, new { Id = jobId, CanceledState = TranscodingJobState.Canceled, DoneState = TranscodingJobState.Done, FailedState = TranscodingJobState.Failed }); scope.Complete(); return(updatedRows > 0); } } }
public bool DeleteJob(Guid jobId) { using (var scope = TransactionUtils.CreateTransactionScope()) { using (var connection = Helper.GetConnection()) { connection.Open(); int rowsDeleted = -1; JobType jobType = GetJobType(jobId, connection); switch (jobType) { case JobType.Audio: case JobType.Mux: rowsDeleted = DeleteMuxJob(jobId, connection); break; case JobType.Video: rowsDeleted = DeleteVideoJob(jobId, connection); break; case JobType.VideoMp4box: case JobType.VideoMerge: throw new NotImplementedException(); case JobType.Unknown: return(false); default: throw new InvalidOperationException(); } scope.Complete(); return(rowsDeleted > 0); } } }
private static void UpdateVideoJob(BaseJob job, IDbConnection connection) { JobRequest jobRequest = connection.Query <JobRequest>( "SELECT JobCorrelationId, VideoSourceFilename, AudioSourceFilename, DestinationFilename, Needed, EnableDash, EnablePsnr FROM FfmpegVideoRequest WHERE JobCorrelationId = @Id", new { Id = job.JobCorrelationId }) .SingleOrDefault(); if (jobRequest == null) { throw new ArgumentException($@"Job with correlation id {job.JobCorrelationId} not found"); } jobRequest.Targets = connection.Query <DestinationFormat>( "SELECT JobCorrelationId, Width, Height, VideoBitrate, AudioBitrate FROM FfmpegVideoRequestTargets WHERE JobCorrelationId = @Id;", new { Id = job.JobCorrelationId }) .ToArray(); using (var scope = TransactionUtils.CreateTransactionScope()) { Type jobType = job.GetType(); TranscodingJobState jobState = job.Failed ? TranscodingJobState.Failed : job.Done ? TranscodingJobState.Done : TranscodingJobState.InProgress; if (jobType == typeof(VideoTranscodingJob)) { int updatedRows = connection.Execute( "UPDATE FfmpegVideoJobs SET Progress = @Progress, Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE Id = @Id;", new { Id = job.Id, Progress = job.Progress.TotalSeconds, Heartbeat = DateTimeOffset.UtcNow.UtcDateTime, State = jobState, MachineName = job.MachineName, }); if (updatedRows != 1) { throw new Exception($"Failed to update progress for job id {job.Id}"); } if (jobRequest.EnablePsnr) { foreach (FfmpegPart chunk in ((VideoTranscodingJob)job).Chunks) { connection.Execute( "UPDATE FfmpegVideoParts SET PSNR = @Psnr WHERE Id = @Id;", new { Id = chunk.Id, Psnr = chunk.Psnr }); } } } else if (jobType == typeof(MergeJob)) { int updatedRows = connection.Execute( "UPDATE FfmpegMergeJobs SET Progress = @Progress, Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE Id = @Id;", new { Id = job.Id, Progress = job.Progress.TotalSeconds, Heartbeat = DateTimeOffset.UtcNow.UtcDateTime, State = jobState, MachineName = job.MachineName }); if (updatedRows != 1) { throw new Exception($"Failed to update progress for job id {job.Id}"); } var states = connection.Query <string>( "SELECT State FROM FfmpegMergeJobs WHERE JobCorrelationId = @Id;", new { Id = job.JobCorrelationId }) .Select(value => Enum.Parse(typeof(TranscodingJobState), value)) .Cast <TranscodingJobState>(); if (states.All(x => x == TranscodingJobState.Done)) { string tempFolder = String.Concat(Path.GetDirectoryName(jobRequest.DestinationFilename), Path.DirectorySeparatorChar, jobRequest.JobCorrelationId.ToString("N")); Directory.Delete(tempFolder, true); } } else if (jobType == typeof(Mp4boxJob)) { int updatedRows = connection.Execute( "UPDATE Mp4boxJobs SET Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE JobCorrelationId = @Id;", new { Id = job.JobCorrelationId, Progress = job.Progress.TotalSeconds, Heartbeat = DateTimeOffset.UtcNow.UtcDateTime, State = jobState, MachineName = job.MachineName }); if (updatedRows != 1) { throw new Exception($"Failed to update progress for job id {job.Id}"); } } scope.Complete(); } using (var scope = TransactionUtils.CreateTransactionScope()) { ICollection <TranscodingJobState> totalJobs = connection.Query <TranscodingJobState>( "SELECT State FROM FfmpegVideoJobs WHERE JobCorrelationId = @Id;", new { Id = jobRequest.JobCorrelationId }) .ToList(); if (totalJobs.Any(x => x != TranscodingJobState.Done)) { // Not all transcoding jobs are finished return; } totalJobs = connection.Query <TranscodingJobState>( "SELECT State FROM FfmpegMergeJobs WHERE JobCorrelationId = @Id;", new { Id = jobRequest.JobCorrelationId }) .ToList(); if (totalJobs.Any(x => x != TranscodingJobState.Done)) { // Not all merge jobs are finished return; } string destinationFilename = jobRequest.DestinationFilename; string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(destinationFilename); string fileExtension = Path.GetExtension(destinationFilename); string outputFolder = Path.GetDirectoryName(destinationFilename); if (totalJobs.Count == 0) { QueueMergeJob(job, connection, outputFolder, fileNameWithoutExtension, fileExtension, jobRequest); } else if (jobRequest.EnableDash) { QueueMpegDashMergeJob(job, destinationFilename, connection, jobRequest, fileNameWithoutExtension, outputFolder, fileExtension); } scope.Complete(); } }
public FFmpegTaskDto GetNextJob(string machineName) { if (string.IsNullOrWhiteSpace(machineName)) { throw new ArgumentNullException(nameof(machineName)); } InsertClientHeartbeat(machineName); int timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]); var now = DateTimeOffset.UtcNow; DateTimeOffset timeout = now.Subtract(TimeSpan.FromSeconds(timeoutSeconds)); using (var connection = Helper.GetConnection()) { connection.Open(); IEnumerable <FFmpegJobDto> existingJobs = GetActiveJobs(machineName, connection); if (existingJobs.Any(x => x.Type == JobType.HardSubtitles)) { return(null); } } do { using (var scope = TransactionUtils.CreateTransactionScope(IsolationLevel.Serializable)) { using (var connection = Helper.GetConnection()) { try { var data = new { QueuedState = TranscodingJobState.Queued, InProgressState = TranscodingJobState.InProgress, Timeout = timeout, Timestamp = now }; connection.Open(); var task = connection.QuerySingleOrDefault <FFmpegTaskDto>("sp_GetNextTask", data, commandType: CommandType.StoredProcedure); if (task == null) { return(null); } // Safety check to ensure that the data is being returned correctly in the SQL query if (task.Id < 0 || task.FfmpegJobsId < 0 || string.IsNullOrWhiteSpace(task.Arguments)) { throw new InvalidOperationException("One or more parameters were not set by SQL query."); } scope.Complete(); return(task); } catch (SqlException e) { // Retry in case of deadlocks if (e.Number == 1205) { continue; } throw; } } } } while (true); }
public VideoTranscodingJob GetNextTranscodingJob() { int timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]); DateTime timeout = DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(timeoutSeconds)); using (var scope = TransactionUtils.CreateTransactionScope()) { using (var connection = Helper.GetConnection()) { connection.Open(); var data = connection.Query <dynamic>( "SELECT TOP 1 Id, Arguments, JobCorrelationId FROM FfmpegVideoJobs WHERE State = @QueuedState OR (State = @InProgressState AND HeartBeat < @Heartbeat) ORDER BY Needed ASC, Id ASC;", new { QueuedState = TranscodingJobState.Queued, InProgressState = TranscodingJobState.InProgress, Heartbeat = timeout }) .SingleOrDefault(); if (data == null) { return(null); } var parts = connection.Query <dynamic>( "SELECT Id, JobCorrelationId, Filename, Number, Target, PSNR FROM FfmpegVideoParts WHERE FfmpegVideoJobs_Id = @JobId;", new { JobId = data.Id }); var rowsUpdated = connection.Execute( "UPDATE FfmpegVideoJobs SET State = @State, HeartBeat = @Heartbeat, Started = @Heartbeat WHERE Id = @Id;", new { State = TranscodingJobState.InProgress, Heartbeat = DateTime.UtcNow, Id = data.Id }); if (rowsUpdated == 0) { throw new Exception("Failed to mark row as taken"); } scope.Complete(); var job = new VideoTranscodingJob { Id = Convert.ToInt32(data.Id), Arguments = data.Arguments.Split('|'), JobCorrelationId = data.JobCorrelationId, Chunks = parts.Select(x => new FfmpegPart { Id = x.Id, JobCorrelationId = x.JobCorrelationId, Psnr = x.PSNR, Target = x.Target, Number = x.Number, SourceFilename = x.SourceFilename, Filename = x.Filename, }).ToList() }; return(job); } } }
public Guid Add(MuxJobRequest request, ICollection <FFmpegJob> jobs) { if (request == null) { throw new ArgumentNullException(nameof(request)); } if (jobs == null) { throw new ArgumentNullException(nameof(jobs)); } Guid jobCorrelationId = Guid.NewGuid(); using (var scope = TransactionUtils.CreateTransactionScope()) { using (var connection = new SqlConnection(_connectionString)) { connection.Execute( "INSERT INTO FfmpegMuxRequest (JobCorrelationId, VideoSourceFilename, AudioSourceFilename, DestinationFilename, OutputFolder, Needed, Created) VALUES(@JobCorrelationId, @VideoSourceFilename, @AudioSourceFilename, @DestinationFilename, @OutputFolder, @Needed, @Created);", new { JobCorrelationId = jobCorrelationId, request.VideoSourceFilename, request.AudioSourceFilename, request.Needed, request.DestinationFilename, request.OutputFolder, Created = DateTime.UtcNow }); foreach (MuxJob job in jobs.Select(x => x as MuxJob)) { var jobId = connection.ExecuteScalar <int>( "INSERT INTO FfmpegJobs (JobCorrelationId, Created, Needed, JobState, JobType) VALUES(@JobCorrelationId, @Created, @Needed, @State, @JobType);SELECT @@IDENTITY;", new { JobCorrelationId = jobCorrelationId, Created = DateTimeOffset.UtcNow, job.Needed, State = job.State, JobType = JobType.Mux }); connection.Execute( "INSERT INTO FfmpegTasks (FfmpegJobs_id, Arguments, TaskState, DestinationFilename, DestinationDurationSeconds, VerifyOutput) VALUES(@FfmpegJobsId, @Arguments, @QueuedState, @DestinationFilename, @DestinationDurationSeconds, @VerifyOutput);", new { FfmpegJobsId = jobId, job.Arguments, QueuedState = TranscodingJobState.Queued, job.DestinationFilename, job.DestinationDurationSeconds, VerifyOutput = false }); } } scope.Complete(); return(jobCorrelationId); } }
public Guid Add(AudioJobRequest request, ICollection <AudioTranscodingJob> jobs) { if (request == null) { throw new ArgumentNullException(nameof(request)); } if (jobs == null) { throw new ArgumentNullException(nameof(jobs)); } if (jobs.Count == 0) { throw new ArgumentException("Jobs parameter must contain at least 1 job", nameof(jobs)); } Guid jobCorrelationId = Guid.NewGuid(); using (var scope = TransactionUtils.CreateTransactionScope()) { using (var connection = _helper.GetConnection()) { connection.Execute( "INSERT INTO FfmpegAudioRequest (JobCorrelationId, SourceFilename, DestinationFilename, OutputFolder, Needed, Created) VALUES(@JobCorrelationId, @SourceFilename, @DestinationFilename, @OutputFolder, @Needed, @Created);", new { JobCorrelationId = jobCorrelationId, SourceFilename = string.Join(" ", request.SourceFilenames), request.DestinationFilename, request.Needed, request.OutputFolder, Created = DateTime.UtcNow }); foreach (AudioDestinationFormat target in request.Targets) { connection.Execute( "INSERT INTO FfmpegAudioRequestTargets (JobCorrelationId, Codec, Format, Bitrate) VALUES(@JobCorrelationId, @Codec, @Format, @Bitrate);", new { jobCorrelationId, Codec = target.AudioCodec.ToString(), Format = target.Format.ToString(), target.Bitrate }); } var jobId = connection.ExecuteScalar <int>( "INSERT INTO FfmpegJobs (JobCorrelationId, Created, Needed, JobState, JobType) VALUES(@JobCorrelationId, @Created, @Needed, @JobState, @JobType);SELECT @@IDENTITY;", new { JobCorrelationId = jobCorrelationId, Created = DateTimeOffset.UtcNow, request.Needed, JobState = TranscodingJobState.Queued, JobType = JobType.Audio }); foreach (AudioTranscodingJob transcodingJob in jobs) { connection.Execute( "INSERT INTO FfmpegTasks (FfmpegJobs_id, Arguments, TaskState, DestinationFilename, DestinationDurationSeconds, VerifyOutput) VALUES(@FfmpegJobsId, @Arguments, @TaskState, @DestinationFilename, @DestinationDurationSeconds, @VerifyOutput);", new { FfmpegJobsId = jobId, transcodingJob.Arguments, TaskState = TranscodingJobState.Queued, transcodingJob.DestinationFilename, transcodingJob.DestinationDurationSeconds, VerifyOutput = true }); } } scope.Complete(); return(jobCorrelationId); } }