public PreprocessorTableEntity(PreprocessorJob job, string status)
        {
            this.PartitionKey = job.Job.AssetID;
            this.RowKey       = $"{job.MmmrkURL.NormalizedURL().ToPartitionKey()}-{job.VideoBitrate}";
            this.VideoURL     = job.Mp4URL != null?job.Mp4URL.NormalizedURL() : "";

            this.MMRKURL = job.MmmrkURL.NormalizedURL();
            this.AssetID = job.Job.AssetID;
            this.JobID   = job.Job.JobID;
            this.Status  = status;
        }
        public Task LogPreprocessorAsync(PreprocessorJob job, string status)
        {
            if (this.preprocessorTable == null)
            {
                return(Task.CompletedTask);
            }

            var entity = new PreprocessorTableEntity(job: job, status: status);

            return(Policy.Handle <Exception>()
                   .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(1),
                                      onRetry: (ex, ts) => { Console.WriteLine($"While logging to table: {ex.Message}"); })
                   .ExecuteAsync(() =>
            {
                return preprocessorTable.InsertOrReplaceAsync <PreprocessorTableEntity>(entity);
            }));
        }
Ejemplo n.º 3
0
        private static async Task RunPreprocessorAsync(PreprocessorJob _, Action <Category, string> stdout, Action <Category, string> stderr)
        {
            ExecutionResult output;

            if (_.RunPreprocessorAndUploadMMRK)
            {
                #region Download MP4

                stdout(Category.DownloadMP4, $"Start Download MP4 {_.Mp4URL.AbsoluteUri}");
                output = await _.Mp4URL.DownloadToAsync(_.LocalFile);

                if (output.Success)
                {
                    stdout(Category.DownloadMP4, output.Output);
                }
                else
                {
                    stderr(Category.DownloadMP4, output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.DownloadMP4
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(Category.QueueNotifications, queueOutput.Output);
                    }

                    return;
                }
                stdout(Category.DownloadMP4, $"Finished Download MP4 {_.Mp4URL.AbsoluteUri}");

                #endregion

                #region Pass 1

                //--x264encopts --bframes 2 --b-pyramid none --b-adapt 0 --no-scenecut
                stdout(Category.PreprocessorStep1, $"Start {_.LocalFile.FullName}");

                output = await Utils.RunProcessAsync(
                    prefix : "PREPROCESSOR1",
                    additionalEnvironment : new Dictionary <string, string> {
                    { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" },
                    { "TMPDIR", Environment.CurrentDirectory }
                },
                    fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
                    arguments : new[] {
                    $"--infile {_.LocalFile.FullName}",
                    $"--stats {_.StatsFile.FullName}",
                    $"--pass 1",
                    $"--vbitrate {(_.VideoBitrate)}",
                    $"--gopsize {_.GOPSize}",
                    "--x264encopts",
                    "--keyint 60",
                    "--min-keyint 60",
                    "--no-scenecut"
                }
                    );

                if (output.Success && _.StatsFile.Exists)
                {
                    stdout(Category.PreprocessorStep1, "SUCCESS");
                    // stdout(Category.PreprocessorStep1, output.Output);
                }
                else
                {
                    stderr(Category.PreprocessorStep1, output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.Pass1
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(Category.QueueNotifications, queueOutput.Output);
                    }

                    return;
                }
                stdout(Category.PreprocessorStep1, $"Finished {_.LocalFile.FullName}");

                #endregion

                #region Pass 2
                stdout(Category.PreprocessorStep2, $"Start {_.LocalFile.FullName}");

                output = await Utils.RunProcessAsync(
                    prefix : "PREPROCESSOR2",
                    additionalEnvironment : new Dictionary <string, string> {
                    { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" },
                    { "TMPDIR", Environment.CurrentDirectory }
                },
                    fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
                    //Original arguments: new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {_.VideoBitrate}", $"--gopsize {_.GOPSize}", string.IsNullOrEmpty(_.VideoFilter) ? "" : $"--x264encopts --video-filter {_.VideoFilter}", "--bframes 2", "--b-pyramid none", "--b-adapt 0", "--keyint 60", "--min-keyint 60", "--no-scenecut" }
                    //TEST 2 arguments: new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {(_.VideoBitrate/1000)}K", $"--gopsize {_.GOPSize}", "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut" }
                    //TEST3
                    arguments : new[] {
                    $"--infile {_.LocalFile.FullName}",
                    $"--stats {_.StatsFile.FullName}",
                    $"--outfile {_.MmrkFile.FullName}",
                    $"--pass 2",
                    $"--vbitrate {(_.VideoBitrate)}",
                    $"--gopsize {_.GOPSize}",
                    "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut"
                }
                    );

                if (output.Success && _.MmrkFile.Exists)
                {
                    stdout(Category.PreprocessorStep2, "SUCCESS");
                    // stdout(Category.PreprocessorStep2, output.Output);
                }
                else
                {
                    stderr(Category.PreprocessorStep2, output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.Pass2
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(Category.QueueNotifications, queueOutput.Output);
                    }

                    return;
                }
                stdout(Category.PreprocessorStep2, $"Finished {_.LocalFile.FullName}");

                #endregion

                #region Delete MP4 and statistics files

                // Delete the input MP4 after MMRK is generated
                if (_.LocalFile.Exists)
                {
                    stdout(Category.Main, $"Deleting {_.LocalFile.FullName}");
                    _.LocalFile.Delete();
                }
                if (_.StatsFile.Exists)
                {
                    _.StatsFile.Delete();
                }
                if (File.Exists($"{_.StatsFile.FullName}.mbtree"))
                {
                    File.Delete($"{_.StatsFile.FullName}.mbtree");
                }

                #endregion

                #region Upload MMRK
                stdout(Category.UploadMMRK, $"Start Upload MMRK {_.MmrkFile.FullName}");

                output = await _.MmrkFile.UploadToAsync(_.MmmrkURL);

                if (output.Success)
                {
                    stdout(Category.UploadMMRK, output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Finished,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.UploadMMRK
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(Category.QueueNotifications, queueOutput.Output);
                    }
                }
                else
                {
                    stderr(Category.UploadMMRK, output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.UploadMMRK
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(Category.QueueNotifications, queueOutput.Output);
                    }

                    return;
                }
                stdout(Category.UploadMMRK, $"End Upload MMRK {_.MmrkFile.FullName}");

                #endregion
            }
            else
            {
                #region Download MMRK
                stdout(Category.DownloadMMRK, $"Start Download MMRK {_.MmrkFile.FullName}");

                output = await _.MmmrkURL.DownloadToAsync(_.MmrkFile);

                if (output.Success)
                {
                    stdout(Category.DownloadMMRK, output.Output);
                }
                else
                {
                    stderr(Category.DownloadMMRK, output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationEmbedder
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = EmbedderStage.DownloadMMRK
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(Category.QueueNotifications, queueOutput.Output);
                    }

                    return;
                }
                stdout(Category.DownloadMMRK, $"Finished Download MMRK {_.MmrkFile.FullName}");

                #endregion
            }
        }
        private static async Task RunPreprocessorAsync(PreprocessorJob _, Action <string> stdout, Action <string> stderr)
        {
            ExecutionResult output;

            if (_.RunPreprocessorAndUploadMMRK)
            {
                #region Download MP4
                stdout($"***Start Download MP4 {DateTime.Now.ToString()}");
                output = await _.Mp4URL.DownloadToAsync(_.LocalFile);

                if (output.Success)
                {
                    stdout(output.Output);
                }
                else
                {
                    stderr(output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.DownloadMP4
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(queueOutput.Output);
                    }

                    return;
                }
                stdout($"*** End Download MP4 {DateTime.Now.ToString()}");

                #endregion

                #region Pass 1
                //--x264encopts --bframes 2 --b-pyramid none --b-adapt 0 --no-scenecut
                stdout($"*** Start PREPROCESSOR1  {DateTime.Now.ToString()}");

                output = await Utils.RunProcessAsync(
                    prefix : "PREPROCESSOR1",
                    additionalEnvironment : new Dictionary <string, string> {
                    { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" }
                },
                    fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
                    //Original
                    //arguments: new[] {
                    //    $"--infile {_.LocalFile.FullName}",
                    //    $"--stats {_.StatsFile.FullName}",
                    //    $"--pass 1",
                    //    $"--vbitrate {_.VideoBitrate}",
                    //    $"--gopsize {_.GOPSize}",
                    //    "--x264encopts",
                    //    //"--bframes 2",
                    //    //"--b-pyramid none",
                    //    //"--b-adapt 0",
                    //    "--keyint 60",
                    //    "--min-keyint 60",
                    //    "--no-scenecut"

                    //}

                    //TEST 2
                    //arguments: new[] {
                    //    $"--infile {_.LocalFile.FullName}",
                    //    $"--stats {_.StatsFile.FullName}",
                    //    $"--pass 1",
                    //    $"--vbitrate {(_.VideoBitrate/1000)}K",
                    //    $"--gopsize {_.GOPSize}",
                    //    "--x264encopts",
                    //    "--keyint 60",
                    //    "--min-keyint 60",
                    //    "--no-scenecut"
                    //}
                    //TEST 3
                    arguments : new[] {
                    $"--infile {_.LocalFile.FullName}",
                    $"--stats {_.StatsFile.FullName}",
                    $"--pass 1",
                    $"--vbitrate {(_.VideoBitrate)}",
                    $"--gopsize {_.GOPSize}",
                    "--x264encopts",
                    "--keyint 60",
                    "--min-keyint 60",
                    "--no-scenecut"
                }
                    );

                if (output.Success && _.StatsFile.Exists)
                {
                    stdout(output.Output);
                }
                else
                {
                    stderr(output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.Pass1
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(queueOutput.Output);
                    }

                    return;
                }
                stdout($"*** End PREPROCESSOR1  {DateTime.Now.ToString()}");

                #endregion

                #region Pass 2
                stdout($"***Start PREPROCESSOR2  {DateTime.Now.ToString()}");

                output = await Utils.RunProcessAsync(
                    prefix : "PREPROCESSOR1",
                    additionalEnvironment : new Dictionary <string, string> {
                    { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" }
                },
                    fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
                    //Original
                    //arguments: new[] {
                    //     $"--infile {_.LocalFile.FullName}",
                    //     $"--stats {_.StatsFile.FullName}",
                    //     $"--outfile {_.MmrkFile.FullName}",
                    //     $"--pass 2",
                    //     $"--vbitrate {_.VideoBitrate}", $"--gopsize {_.GOPSize}",
                    //     string.IsNullOrEmpty(_.VideoFilter) ? "" : $"--x264encopts --video-filter {_.VideoFilter}",
                    //     //"--bframes 2",
                    //     //"--b-pyramid none",
                    //     //"--b-adapt 0",
                    //      "--keyint 60",
                    //     "--min-keyint 60",
                    //     "--no-scenecut"
                    //}
                    //TEST 2
                    //arguments: new[] {
                    //     $"--infile {_.LocalFile.FullName}",
                    //     $"--stats {_.StatsFile.FullName}",
                    //     $"--outfile {_.MmrkFile.FullName}",
                    //     $"--pass 2",
                    //     $"--vbitrate {(_.VideoBitrate/1000)}K",
                    //     $"--gopsize {_.GOPSize}",
                    //     "--x264encopts",
                    //     "--keyint 60",
                    //    "--min-keyint 60",
                    //    "--no-scenecut"
                    //}
                    //TEST3
                    arguments : new[] {
                    $"--infile {_.LocalFile.FullName}",
                    $"--stats {_.StatsFile.FullName}",
                    $"--outfile {_.MmrkFile.FullName}",
                    $"--pass 2",
                    $"--vbitrate {(_.VideoBitrate)}",
                    $"--gopsize {_.GOPSize}",
                    "--x264encopts",
                    "--keyint 60",
                    "--min-keyint 60",
                    "--no-scenecut"
                }
                    );

                if (output.Success && _.MmrkFile.Exists)
                {
                    stdout(output.Output);
                }
                else
                {
                    stderr(output.Output);
                    stdout($"***Start  PREPROCESSOR Error Notification Message {DateTime.Now.ToString()}");
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.Pass2
                    });

                    stdout($"***End  PREPROCESSOR  Error Notification Message {DateTime.Now.ToString()}");


                    if (!queueOutput.Success)
                    {
                        stderr(queueOutput.Output);
                    }

                    return;
                }
                stdout($"***End PREPROCESSOR2 {DateTime.Now.ToString()}");

                #endregion

                #region Upload MMRK
                stdout($"***Start Upload MMRK {DateTime.Now.ToString()}");

                output = await _.MmrkFile.UploadToAsync(_.MmmrkURL);

                if (output.Success)
                {
                    stdout(output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Finished,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.UploadMMRK
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(queueOutput.Output);
                    }
                }
                else
                {
                    stderr(output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = PreprocessorStage.UploadMMRK
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(queueOutput.Output);
                    }

                    return;
                }
                stdout($"***End Upload MMRK {DateTime.Now.ToString()}");

                #endregion
            }
            else
            {
                #region Download MMRK
                stdout($"***Start Download MMRK MMRK {DateTime.Now.ToString()}");

                output = await _.MmmrkURL.DownloadToAsync(_.MmrkFile);

                if (output.Success)
                {
                    stdout(output.Output);
                }
                else
                {
                    stderr(output.Output);
                    var queueOutput = await _.Queue.DispatchMessage(new NotificationEmbedder
                    {
                        AssetID   = _.Job.AssetID,
                        JobID     = _.Job.JobID,
                        FileName  = _.Name,
                        Status    = JobStatus.Error,
                        JobOutput = output.Output,
                        Stage     = EmbedderStage.DownloadMMRK
                    });

                    if (!queueOutput.Success)
                    {
                        stderr(queueOutput.Output);
                    }

                    return;
                }
                stdout($"***End Download MMRK MMRK {DateTime.Now.ToString()}");

                #endregion
            }
        }
Ejemplo n.º 5
0
        static async Task <int> MainAsync(string[] args)
        {
            #region Install licenses

            if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("LICENSES")))
            {
                stderr(Category.Main, "Could not determine licenses from environment...");
                return(-1);
            }
            LicenseData.InjectIntoFilesystem(environmentVariable: "LICENSES");

            #endregion

            var tableConnectionString = Environment.GetEnvironmentVariable("LOGGINGTABLE");
            if (string.IsNullOrEmpty(tableConnectionString))
            {
                stdout(Program.Category.Main, $"Could not configure table logging, missing environment variable 'LOGGINGTABLE'");
            }
            var table = await TableWrapper.CreateAsync(
                connectionString : tableConnectionString,
                preprocessorTableName : "preprocessor",
                embedderTableName : "embedder");

            #region Read Job

            EmbedderJobDTO job = null;
            var            jobEnvironmentVariable = Environment.GetEnvironmentVariable("JOB");
            if (string.IsNullOrEmpty(jobEnvironmentVariable))
            {
                stderr(Category.Main, "Could not retrieve job from 'JOB' environment variable...");
                return(-1);
            }
            try
            {
                string json = null;
                if (jobEnvironmentVariable.StartsWith("http"))
                {
                    var ms = new MemoryStream();
                    using (var client = new HttpClient())
                        using (var stream = await client.GetStreamAsync(jobEnvironmentVariable))
                        {
                            await stream.CopyToAsync(ms);
                        }
                    json = Encoding.UTF8.GetString(ms.GetBuffer());
                }
                else
                {
                    json = Encoding.UTF8.GetString(
                        Convert.FromBase64String(jobEnvironmentVariable));
                }

                // cat juan.json | jq -c -M . | base64 --wrap=0 > juan.base64
                job = JsonConvert.DeserializeObject <EmbedderJobDTO>(json);
                if (job == null)
                {
                    stderr(Category.Main, "Could not read job description from 'JOB' environment variable");
                    return(-1);
                }
            }
            catch (Exception ex)
            {
                stderr(Category.Main, $"Could not parse the job description: {ex.Message}");
                stderr(Category.Main, $"JOB == {jobEnvironmentVariable}");
                return(-1);
            }

            #endregion

            var workFolder =
                Path.Combine("/mnt", job.JobID
                             .Replace(":", "_")
                             .Replace("/", "_"));

            stdout(Category.Main, $"Changing work directory to {workFolder}");

            Directory.CreateDirectory(workFolder);
            Environment.CurrentDirectory = workFolder;

            #region MMRK Generation

            var preprocessorData = PreprocessorJob.DeterminePreprocessorJobs(job);
            stdout(Category.Main, "Start Preprocessor");
            foreach (var pd in preprocessorData)
            {
                // run these compute-intensive jobs sequentially
                await Program.RunPreprocessorAsync(pd, stdout, stderr, table);
            }
            stdout(Category.Main, "Finished Preprocessor");

            #endregion

            #region Watermarking

            Func <int> getNumberOfParallelEmbedderTasks = () =>
            {
                int result;
                if (int.TryParse(Environment.GetEnvironmentVariable("PARALLELEMBEDDERS"), out result))
                {
                    return(result);
                }
                return(5);
            };

            var parallelEmbedderTasks = getNumberOfParallelEmbedderTasks();

            stdout(Category.Main, "Start Embedder");
            var embedderData = EmbedderJob.DetermineEmbedderJobs(job);

            await embedderData.ForEachAsync(
                parallelTasks : parallelEmbedderTasks,
                task : _ => Program.RunEmbedderAsync(_, stdout, stderr, table));

            stdout(Category.Main, "Finished Embedder");

            #endregion

            #region Delete all MMRK files from pod filesystem

            foreach (var pd in preprocessorData)
            {
                Policy
                .Handle <Exception>()
                .WaitAndRetry(
                    retryCount: 5,
                    sleepDurationProvider: attempt => TimeSpan.FromSeconds(1))
                .Execute(() =>
                {
                    if (pd.MmrkFile.Exists)
                    {
                        pd.MmrkFile.Delete();
                    }
                });
            }

            #endregion

            Environment.CurrentDirectory = "/";
            stdout(Category.Main, $"Removing work directory {workFolder}");
            Directory.Delete(workFolder, recursive: true);

            return(0);
        }