public Mp4boxJob GetNextDashJob()
        {
            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var connection = Helper.GetConnection())
                {
                    connection.Open();

                    var data = connection.Query <Mp4boxJob>(
                        "SELECT TOP 1 JobCorrelationId, Arguments FROM Mp4boxJobs WHERE State = @State ORDER BY Needed ASC, Id ASC;",
                        new { State = TranscodingJobState.Queued })
                               .SingleOrDefault();
                    if (data == null)
                    {
                        return(null);
                    }

                    var rowsUpdated =
                        connection.Execute("UPDATE Mp4boxJobs SET State = @State WHERE JobCorrelationId = @Id;",
                                           new { State = TranscodingJobState.InProgress, Id = data.JobCorrelationId });
                    if (rowsUpdated != 1)
                    {
                        return(null);
                    }

                    scope.Complete();

                    return(new Mp4boxJob
                    {
                        JobCorrelationId = data.JobCorrelationId,
                        Arguments = data.Arguments
                    });
                }
            }
        }
        public bool PauseJob(Guid jobId)
        {
            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var conn = Helper.GetConnection())
                {
                    JobType jobType = GetJobType(jobId, conn);

                    string sql = "UPDATE FfmpegJobs SET JobState = @PausedState WHERE JobCorrelationId = @Id AND JobState = @QueuedState;";
                    switch (jobType)
                    {
                    case JobType.Audio:
                    case JobType.Mux:
                        sql += "UPDATE Tasks SET Tasks.TaskState = @PausedState FROM FfmpegTasks Tasks INNER JOIN FfmpegJobs Jobs ON Jobs.id = Tasks.FfmpegJobs_Id WHERE Jobs.JobCorrelationId = @Id AND Tasks.TaskState = @QueuedState;";
                        break;

                    case JobType.Video:
                    case JobType.VideoMp4box:
                    case JobType.VideoMerge:
                        throw new NotImplementedException();

                    case JobType.Unknown:
                    default:
                        throw new ArgumentOutOfRangeException();
                    }
                    int updatedRows = conn.Execute(sql,
                                                   new { Id = jobId, PausedState = TranscodingJobState.Paused, QueuedState = TranscodingJobState.Queued });

                    scope.Complete();

                    return(updatedRows > 0);
                }
            }
        }
 public int PruneInactiveClients(TimeSpan maxAge)
 {
     using (var scope = TransactionUtils.CreateTransactionScope(IsolationLevel.Serializable))
     {
         using (var connection = Helper.GetConnection())
         {
             var res = connection.Execute("DELETE FROM Clients WHERE Clients.LastHeartbeat < @MaxAge",
                                          new { MaxAge = DateTimeOffset.Now - maxAge });
             scope.Complete();
             return(res);
         }
     }
 }
        public MergeJob GetNextMergeJob()
        {
            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var connection = Helper.GetConnection())
                {
                    connection.Open();

                    int      timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]);
                    DateTime timeout        = DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(timeoutSeconds));

                    var data = connection.Query <dynamic>(
                        "SELECT TOP 1 Id, Arguments, JobCorrelationId FROM FfmpegMergeJobs WHERE State = @QueuedState OR (State = @InProgressState AND HeartBeat < @Heartbeat) ORDER BY Needed ASC, Id ASC;",
                        new
                    {
                        QueuedState     = TranscodingJobState.Queued,
                        InProgressState = TranscodingJobState.InProgress,
                        Heartbeat       = timeout
                    })
                               .SingleOrDefault();
                    if (data == null)
                    {
                        return(null);
                    }

                    var rowsUpdated = connection.Execute(
                        "UPDATE FfmpegMergeJobs SET State = @State, HeartBeat = @Heartbeat WHERE Id = @Id;",
                        new { State = TranscodingJobState.InProgress, Heartbeat = DateTime.UtcNow, Id = data.Id });
                    if (rowsUpdated == 0)
                    {
                        throw new Exception("Failed to mark row as taken");
                    }

                    scope.Complete();

                    return(new MergeJob
                    {
                        Id = Convert.ToInt32(data.Id),
                        Arguments = new string[] { data.Arguments },
                        JobCorrelationId = data.JobCorrelationId
                    });
                }
            }
        }
        public bool CancelJob(Guid jobId)
        {
            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var conn = Helper.GetConnection())
                {
                    string sql = "UPDATE FfmpegJobs SET JobState = @CanceledState WHERE JobCorrelationId = @Id AND jobState <> @DoneState AND jobState <> @FailedState;";
                    sql += "UPDATE Tasks SET Tasks.TaskState = @CanceledState FROM FfmpegTasks Tasks INNER JOIN FfmpegJobs Jobs ON Jobs.id = Tasks.FfmpegJobs_Id " +
                           "WHERE Jobs.JobCorrelationId = @Id AND Tasks.TaskState <> @DoneState AND Tasks.TaskState <> @FailedState;";

                    int updatedRows = conn.Execute(sql,
                                                   new { Id = jobId, CanceledState = TranscodingJobState.Canceled, DoneState = TranscodingJobState.Done, FailedState = TranscodingJobState.Failed });

                    scope.Complete();

                    return(updatedRows > 0);
                }
            }
        }
        public bool DeleteJob(Guid jobId)
        {
            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var connection = Helper.GetConnection())
                {
                    connection.Open();

                    int rowsDeleted = -1;

                    JobType jobType = GetJobType(jobId, connection);

                    switch (jobType)
                    {
                    case JobType.Audio:
                    case JobType.Mux:
                        rowsDeleted = DeleteMuxJob(jobId, connection);
                        break;

                    case JobType.Video:
                        rowsDeleted = DeleteVideoJob(jobId, connection);
                        break;

                    case JobType.VideoMp4box:
                    case JobType.VideoMerge:
                        throw new NotImplementedException();

                    case JobType.Unknown:
                        return(false);

                    default:
                        throw new InvalidOperationException();
                    }

                    scope.Complete();

                    return(rowsDeleted > 0);
                }
            }
        }
        private static void UpdateVideoJob(BaseJob job, IDbConnection connection)
        {
            JobRequest jobRequest = connection.Query <JobRequest>(
                "SELECT JobCorrelationId, VideoSourceFilename, AudioSourceFilename, DestinationFilename, Needed, EnableDash, EnablePsnr FROM FfmpegVideoRequest WHERE JobCorrelationId = @Id",
                new { Id = job.JobCorrelationId })
                                    .SingleOrDefault();

            if (jobRequest == null)
            {
                throw new ArgumentException($@"Job with correlation id {job.JobCorrelationId} not found");
            }

            jobRequest.Targets = connection.Query <DestinationFormat>(
                "SELECT JobCorrelationId, Width, Height, VideoBitrate, AudioBitrate FROM FfmpegVideoRequestTargets WHERE JobCorrelationId = @Id;",
                new { Id = job.JobCorrelationId })
                                 .ToArray();

            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                Type jobType = job.GetType();
                TranscodingJobState jobState = job.Failed
                    ? TranscodingJobState.Failed
                    : job.Done
                        ? TranscodingJobState.Done
                        : TranscodingJobState.InProgress;

                if (jobType == typeof(VideoTranscodingJob))
                {
                    int updatedRows = connection.Execute(
                        "UPDATE FfmpegVideoJobs SET Progress = @Progress, Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE Id = @Id;",
                        new
                    {
                        Id          = job.Id,
                        Progress    = job.Progress.TotalSeconds,
                        Heartbeat   = DateTimeOffset.UtcNow.UtcDateTime,
                        State       = jobState,
                        MachineName = job.MachineName,
                    });

                    if (updatedRows != 1)
                    {
                        throw new Exception($"Failed to update progress for job id {job.Id}");
                    }

                    if (jobRequest.EnablePsnr)
                    {
                        foreach (FfmpegPart chunk in ((VideoTranscodingJob)job).Chunks)
                        {
                            connection.Execute(
                                "UPDATE FfmpegVideoParts SET PSNR = @Psnr WHERE Id = @Id;",
                                new { Id = chunk.Id, Psnr = chunk.Psnr });
                        }
                    }
                }
                else if (jobType == typeof(MergeJob))
                {
                    int updatedRows = connection.Execute(
                        "UPDATE FfmpegMergeJobs SET Progress = @Progress, Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE Id = @Id;",
                        new
                    {
                        Id          = job.Id,
                        Progress    = job.Progress.TotalSeconds,
                        Heartbeat   = DateTimeOffset.UtcNow.UtcDateTime,
                        State       = jobState,
                        MachineName = job.MachineName
                    });

                    if (updatedRows != 1)
                    {
                        throw new Exception($"Failed to update progress for job id {job.Id}");
                    }

                    var states = connection.Query <string>(
                        "SELECT State FROM FfmpegMergeJobs WHERE JobCorrelationId = @Id;",
                        new { Id = job.JobCorrelationId })
                                 .Select(value => Enum.Parse(typeof(TranscodingJobState), value))
                                 .Cast <TranscodingJobState>();

                    if (states.All(x => x == TranscodingJobState.Done))
                    {
                        string tempFolder = String.Concat(Path.GetDirectoryName(jobRequest.DestinationFilename),
                                                          Path.DirectorySeparatorChar, jobRequest.JobCorrelationId.ToString("N"));

                        Directory.Delete(tempFolder, true);
                    }
                }
                else if (jobType == typeof(Mp4boxJob))
                {
                    int updatedRows = connection.Execute(
                        "UPDATE Mp4boxJobs SET Heartbeat = @Heartbeat, State = @State, HeartbeatMachineName = @MachineName WHERE JobCorrelationId = @Id;",
                        new
                    {
                        Id          = job.JobCorrelationId,
                        Progress    = job.Progress.TotalSeconds,
                        Heartbeat   = DateTimeOffset.UtcNow.UtcDateTime,
                        State       = jobState,
                        MachineName = job.MachineName
                    });

                    if (updatedRows != 1)
                    {
                        throw new Exception($"Failed to update progress for job id {job.Id}");
                    }
                }

                scope.Complete();
            }

            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                ICollection <TranscodingJobState> totalJobs = connection.Query <TranscodingJobState>(
                    "SELECT State FROM FfmpegVideoJobs WHERE JobCorrelationId = @Id;",
                    new { Id = jobRequest.JobCorrelationId })
                                                              .ToList();

                if (totalJobs.Any(x => x != TranscodingJobState.Done))
                {
                    // Not all transcoding jobs are finished
                    return;
                }

                totalJobs = connection.Query <TranscodingJobState>(
                    "SELECT State FROM FfmpegMergeJobs WHERE JobCorrelationId = @Id;",
                    new { Id = jobRequest.JobCorrelationId })
                            .ToList();
                if (totalJobs.Any(x => x != TranscodingJobState.Done))
                {
                    // Not all merge jobs are finished
                    return;
                }

                string destinationFilename      = jobRequest.DestinationFilename;
                string fileNameWithoutExtension =
                    Path.GetFileNameWithoutExtension(destinationFilename);
                string fileExtension = Path.GetExtension(destinationFilename);
                string outputFolder  = Path.GetDirectoryName(destinationFilename);

                if (totalJobs.Count == 0)
                {
                    QueueMergeJob(job, connection, outputFolder, fileNameWithoutExtension, fileExtension, jobRequest);
                }
                else if (jobRequest.EnableDash)
                {
                    QueueMpegDashMergeJob(job, destinationFilename, connection, jobRequest, fileNameWithoutExtension,
                                          outputFolder, fileExtension);
                }

                scope.Complete();
            }
        }
        public FFmpegTaskDto GetNextJob(string machineName)
        {
            if (string.IsNullOrWhiteSpace(machineName))
            {
                throw new ArgumentNullException(nameof(machineName));
            }

            InsertClientHeartbeat(machineName);

            int timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]);
            var now            = DateTimeOffset.UtcNow;

            DateTimeOffset timeout = now.Subtract(TimeSpan.FromSeconds(timeoutSeconds));

            using (var connection = Helper.GetConnection())
            {
                connection.Open();

                IEnumerable <FFmpegJobDto> existingJobs = GetActiveJobs(machineName, connection);
                if (existingJobs.Any(x => x.Type == JobType.HardSubtitles))
                {
                    return(null);
                }
            }

            do
            {
                using (var scope = TransactionUtils.CreateTransactionScope(IsolationLevel.Serializable))
                {
                    using (var connection = Helper.GetConnection())
                    {
                        try
                        {
                            var data = new
                            {
                                QueuedState     = TranscodingJobState.Queued,
                                InProgressState = TranscodingJobState.InProgress,
                                Timeout         = timeout,
                                Timestamp       = now
                            };

                            connection.Open();

                            var task = connection.QuerySingleOrDefault <FFmpegTaskDto>("sp_GetNextTask", data, commandType: CommandType.StoredProcedure);
                            if (task == null)
                            {
                                return(null);
                            }

                            // Safety check to ensure that the data is being returned correctly in the SQL query
                            if (task.Id < 0 || task.FfmpegJobsId < 0 || string.IsNullOrWhiteSpace(task.Arguments))
                            {
                                throw new InvalidOperationException("One or more parameters were not set by SQL query.");
                            }

                            scope.Complete();

                            return(task);
                        }
                        catch (SqlException e)
                        {
                            // Retry in case of deadlocks
                            if (e.Number == 1205)
                            {
                                continue;
                            }

                            throw;
                        }
                    }
                }
            } while (true);
        }
        public VideoTranscodingJob GetNextTranscodingJob()
        {
            int      timeoutSeconds = Convert.ToInt32(ConfigurationManager.AppSettings["TimeoutSeconds"]);
            DateTime timeout        =
                DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(timeoutSeconds));

            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var connection = Helper.GetConnection())
                {
                    connection.Open();

                    var data = connection.Query <dynamic>(
                        "SELECT TOP 1 Id, Arguments, JobCorrelationId FROM FfmpegVideoJobs WHERE State = @QueuedState OR (State = @InProgressState AND HeartBeat < @Heartbeat) ORDER BY Needed ASC, Id ASC;",
                        new
                    {
                        QueuedState     = TranscodingJobState.Queued,
                        InProgressState = TranscodingJobState.InProgress,
                        Heartbeat       = timeout
                    })
                               .SingleOrDefault();
                    if (data == null)
                    {
                        return(null);
                    }

                    var parts =
                        connection.Query <dynamic>(
                            "SELECT Id, JobCorrelationId, Filename, Number, Target, PSNR FROM FfmpegVideoParts WHERE FfmpegVideoJobs_Id = @JobId;",
                            new { JobId = data.Id });

                    var rowsUpdated =
                        connection.Execute(
                            "UPDATE FfmpegVideoJobs SET State = @State, HeartBeat = @Heartbeat, Started = @Heartbeat WHERE Id = @Id;",
                            new { State = TranscodingJobState.InProgress, Heartbeat = DateTime.UtcNow, Id = data.Id });
                    if (rowsUpdated == 0)
                    {
                        throw new Exception("Failed to mark row as taken");
                    }

                    scope.Complete();

                    var job = new VideoTranscodingJob
                    {
                        Id               = Convert.ToInt32(data.Id),
                        Arguments        = data.Arguments.Split('|'),
                        JobCorrelationId = data.JobCorrelationId,
                        Chunks           = parts.Select(x => new FfmpegPart
                        {
                            Id = x.Id,
                            JobCorrelationId = x.JobCorrelationId,
                            Psnr             = x.PSNR,
                            Target           = x.Target,
                            Number           = x.Number,
                            SourceFilename   = x.SourceFilename,
                            Filename         = x.Filename,
                        }).ToList()
                    };
                    return(job);
                }
            }
        }
Beispiel #10
0
        public Guid Add(MuxJobRequest request, ICollection <FFmpegJob> jobs)
        {
            if (request == null)
            {
                throw new ArgumentNullException(nameof(request));
            }
            if (jobs == null)
            {
                throw new ArgumentNullException(nameof(jobs));
            }

            Guid jobCorrelationId = Guid.NewGuid();

            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var connection = new SqlConnection(_connectionString))
                {
                    connection.Execute(
                        "INSERT INTO FfmpegMuxRequest (JobCorrelationId, VideoSourceFilename, AudioSourceFilename, DestinationFilename, OutputFolder, Needed, Created) VALUES(@JobCorrelationId, @VideoSourceFilename, @AudioSourceFilename, @DestinationFilename, @OutputFolder, @Needed, @Created);",
                        new
                    {
                        JobCorrelationId = jobCorrelationId,
                        request.VideoSourceFilename,
                        request.AudioSourceFilename,
                        request.Needed,
                        request.DestinationFilename,
                        request.OutputFolder,
                        Created = DateTime.UtcNow
                    });

                    foreach (MuxJob job in jobs.Select(x => x as MuxJob))
                    {
                        var jobId = connection.ExecuteScalar <int>(
                            "INSERT INTO FfmpegJobs (JobCorrelationId, Created, Needed, JobState, JobType) VALUES(@JobCorrelationId, @Created, @Needed, @State, @JobType);SELECT @@IDENTITY;",
                            new
                        {
                            JobCorrelationId = jobCorrelationId,
                            Created          = DateTimeOffset.UtcNow,
                            job.Needed,
                            State   = job.State,
                            JobType = JobType.Mux
                        });

                        connection.Execute(
                            "INSERT INTO FfmpegTasks (FfmpegJobs_id, Arguments, TaskState, DestinationFilename, DestinationDurationSeconds, VerifyOutput) VALUES(@FfmpegJobsId, @Arguments, @QueuedState, @DestinationFilename, @DestinationDurationSeconds, @VerifyOutput);",
                            new
                        {
                            FfmpegJobsId = jobId,
                            job.Arguments,
                            QueuedState = TranscodingJobState.Queued,
                            job.DestinationFilename,
                            job.DestinationDurationSeconds,
                            VerifyOutput = false
                        });
                    }
                }

                scope.Complete();

                return(jobCorrelationId);
            }
        }
Beispiel #11
0
        public Guid Add(AudioJobRequest request, ICollection <AudioTranscodingJob> jobs)
        {
            if (request == null)
            {
                throw new ArgumentNullException(nameof(request));
            }
            if (jobs == null)
            {
                throw new ArgumentNullException(nameof(jobs));
            }
            if (jobs.Count == 0)
            {
                throw new ArgumentException("Jobs parameter must contain at least 1 job", nameof(jobs));
            }

            Guid jobCorrelationId = Guid.NewGuid();

            using (var scope = TransactionUtils.CreateTransactionScope())
            {
                using (var connection = _helper.GetConnection())
                {
                    connection.Execute(
                        "INSERT INTO FfmpegAudioRequest (JobCorrelationId, SourceFilename, DestinationFilename, OutputFolder, Needed, Created) VALUES(@JobCorrelationId, @SourceFilename, @DestinationFilename, @OutputFolder, @Needed, @Created);",
                        new
                    {
                        JobCorrelationId = jobCorrelationId,
                        SourceFilename   = string.Join(" ", request.SourceFilenames),
                        request.DestinationFilename,
                        request.Needed,
                        request.OutputFolder,
                        Created = DateTime.UtcNow
                    });

                    foreach (AudioDestinationFormat target in request.Targets)
                    {
                        connection.Execute(
                            "INSERT INTO FfmpegAudioRequestTargets (JobCorrelationId, Codec, Format, Bitrate) VALUES(@JobCorrelationId, @Codec, @Format, @Bitrate);",
                            new
                        {
                            jobCorrelationId,
                            Codec  = target.AudioCodec.ToString(),
                            Format = target.Format.ToString(),
                            target.Bitrate
                        });
                    }

                    var jobId = connection.ExecuteScalar <int>(
                        "INSERT INTO FfmpegJobs (JobCorrelationId, Created, Needed, JobState, JobType) VALUES(@JobCorrelationId, @Created, @Needed, @JobState, @JobType);SELECT @@IDENTITY;",
                        new
                    {
                        JobCorrelationId = jobCorrelationId,
                        Created          = DateTimeOffset.UtcNow,
                        request.Needed,
                        JobState = TranscodingJobState.Queued,
                        JobType  = JobType.Audio
                    });

                    foreach (AudioTranscodingJob transcodingJob in jobs)
                    {
                        connection.Execute(
                            "INSERT INTO FfmpegTasks (FfmpegJobs_id, Arguments, TaskState, DestinationFilename, DestinationDurationSeconds, VerifyOutput) VALUES(@FfmpegJobsId, @Arguments, @TaskState, @DestinationFilename, @DestinationDurationSeconds, @VerifyOutput);",
                            new
                        {
                            FfmpegJobsId = jobId,
                            transcodingJob.Arguments,
                            TaskState = TranscodingJobState.Queued,
                            transcodingJob.DestinationFilename,
                            transcodingJob.DestinationDurationSeconds,
                            VerifyOutput = true
                        });
                    }
                }

                scope.Complete();

                return(jobCorrelationId);
            }
        }