public PreprocessorTableEntity(PreprocessorJob job, string status) { this.PartitionKey = job.Job.AssetID; this.RowKey = $"{job.MmmrkURL.NormalizedURL().ToPartitionKey()}-{job.VideoBitrate}"; this.VideoURL = job.Mp4URL != null?job.Mp4URL.NormalizedURL() : ""; this.MMRKURL = job.MmmrkURL.NormalizedURL(); this.AssetID = job.Job.AssetID; this.JobID = job.Job.JobID; this.Status = status; }
public Task LogPreprocessorAsync(PreprocessorJob job, string status) { if (this.preprocessorTable == null) { return(Task.CompletedTask); } var entity = new PreprocessorTableEntity(job: job, status: status); return(Policy.Handle <Exception>() .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(1), onRetry: (ex, ts) => { Console.WriteLine($"While logging to table: {ex.Message}"); }) .ExecuteAsync(() => { return preprocessorTable.InsertOrReplaceAsync <PreprocessorTableEntity>(entity); })); }
private static async Task RunPreprocessorAsync(PreprocessorJob _, Action <Category, string> stdout, Action <Category, string> stderr) { ExecutionResult output; if (_.RunPreprocessorAndUploadMMRK) { #region Download MP4 stdout(Category.DownloadMP4, $"Start Download MP4 {_.Mp4URL.AbsoluteUri}"); output = await _.Mp4URL.DownloadToAsync(_.LocalFile); if (output.Success) { stdout(Category.DownloadMP4, output.Output); } else { stderr(Category.DownloadMP4, output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.DownloadMP4 }); if (!queueOutput.Success) { stderr(Category.QueueNotifications, queueOutput.Output); } return; } stdout(Category.DownloadMP4, $"Finished Download MP4 {_.Mp4URL.AbsoluteUri}"); #endregion #region Pass 1 //--x264encopts --bframes 2 --b-pyramid none --b-adapt 0 --no-scenecut stdout(Category.PreprocessorStep1, $"Start {_.LocalFile.FullName}"); output = await Utils.RunProcessAsync( prefix : "PREPROCESSOR1", additionalEnvironment : new Dictionary <string, string> { { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" }, { "TMPDIR", Environment.CurrentDirectory } }, fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor", arguments : new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--pass 1", $"--vbitrate {(_.VideoBitrate)}", $"--gopsize {_.GOPSize}", "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut" } ); if (output.Success && _.StatsFile.Exists) { stdout(Category.PreprocessorStep1, "SUCCESS"); // stdout(Category.PreprocessorStep1, output.Output); } else { stderr(Category.PreprocessorStep1, output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.Pass1 }); if (!queueOutput.Success) { stderr(Category.QueueNotifications, queueOutput.Output); } return; } stdout(Category.PreprocessorStep1, $"Finished {_.LocalFile.FullName}"); #endregion #region Pass 2 stdout(Category.PreprocessorStep2, $"Start {_.LocalFile.FullName}"); output = await Utils.RunProcessAsync( prefix : "PREPROCESSOR2", additionalEnvironment : new Dictionary <string, string> { { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" }, { "TMPDIR", Environment.CurrentDirectory } }, fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor", //Original arguments: new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {_.VideoBitrate}", $"--gopsize {_.GOPSize}", string.IsNullOrEmpty(_.VideoFilter) ? "" : $"--x264encopts --video-filter {_.VideoFilter}", "--bframes 2", "--b-pyramid none", "--b-adapt 0", "--keyint 60", "--min-keyint 60", "--no-scenecut" } //TEST 2 arguments: new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {(_.VideoBitrate/1000)}K", $"--gopsize {_.GOPSize}", "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut" } //TEST3 arguments : new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {(_.VideoBitrate)}", $"--gopsize {_.GOPSize}", "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut" } ); if (output.Success && _.MmrkFile.Exists) { stdout(Category.PreprocessorStep2, "SUCCESS"); // stdout(Category.PreprocessorStep2, output.Output); } else { stderr(Category.PreprocessorStep2, output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.Pass2 }); if (!queueOutput.Success) { stderr(Category.QueueNotifications, queueOutput.Output); } return; } stdout(Category.PreprocessorStep2, $"Finished {_.LocalFile.FullName}"); #endregion #region Delete MP4 and statistics files // Delete the input MP4 after MMRK is generated if (_.LocalFile.Exists) { stdout(Category.Main, $"Deleting {_.LocalFile.FullName}"); _.LocalFile.Delete(); } if (_.StatsFile.Exists) { _.StatsFile.Delete(); } if (File.Exists($"{_.StatsFile.FullName}.mbtree")) { File.Delete($"{_.StatsFile.FullName}.mbtree"); } #endregion #region Upload MMRK stdout(Category.UploadMMRK, $"Start Upload MMRK {_.MmrkFile.FullName}"); output = await _.MmrkFile.UploadToAsync(_.MmmrkURL); if (output.Success) { stdout(Category.UploadMMRK, output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Finished, JobOutput = output.Output, Stage = PreprocessorStage.UploadMMRK }); if (!queueOutput.Success) { stderr(Category.QueueNotifications, queueOutput.Output); } } else { stderr(Category.UploadMMRK, output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.UploadMMRK }); if (!queueOutput.Success) { stderr(Category.QueueNotifications, queueOutput.Output); } return; } stdout(Category.UploadMMRK, $"End Upload MMRK {_.MmrkFile.FullName}"); #endregion } else { #region Download MMRK stdout(Category.DownloadMMRK, $"Start Download MMRK {_.MmrkFile.FullName}"); output = await _.MmmrkURL.DownloadToAsync(_.MmrkFile); if (output.Success) { stdout(Category.DownloadMMRK, output.Output); } else { stderr(Category.DownloadMMRK, output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationEmbedder { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = EmbedderStage.DownloadMMRK }); if (!queueOutput.Success) { stderr(Category.QueueNotifications, queueOutput.Output); } return; } stdout(Category.DownloadMMRK, $"Finished Download MMRK {_.MmrkFile.FullName}"); #endregion } }
private static async Task RunPreprocessorAsync(PreprocessorJob _, Action <string> stdout, Action <string> stderr) { ExecutionResult output; if (_.RunPreprocessorAndUploadMMRK) { #region Download MP4 stdout($"***Start Download MP4 {DateTime.Now.ToString()}"); output = await _.Mp4URL.DownloadToAsync(_.LocalFile); if (output.Success) { stdout(output.Output); } else { stderr(output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.DownloadMP4 }); if (!queueOutput.Success) { stderr(queueOutput.Output); } return; } stdout($"*** End Download MP4 {DateTime.Now.ToString()}"); #endregion #region Pass 1 //--x264encopts --bframes 2 --b-pyramid none --b-adapt 0 --no-scenecut stdout($"*** Start PREPROCESSOR1 {DateTime.Now.ToString()}"); output = await Utils.RunProcessAsync( prefix : "PREPROCESSOR1", additionalEnvironment : new Dictionary <string, string> { { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" } }, fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor", //Original //arguments: new[] { // $"--infile {_.LocalFile.FullName}", // $"--stats {_.StatsFile.FullName}", // $"--pass 1", // $"--vbitrate {_.VideoBitrate}", // $"--gopsize {_.GOPSize}", // "--x264encopts", // //"--bframes 2", // //"--b-pyramid none", // //"--b-adapt 0", // "--keyint 60", // "--min-keyint 60", // "--no-scenecut" //} //TEST 2 //arguments: new[] { // $"--infile {_.LocalFile.FullName}", // $"--stats {_.StatsFile.FullName}", // $"--pass 1", // $"--vbitrate {(_.VideoBitrate/1000)}K", // $"--gopsize {_.GOPSize}", // "--x264encopts", // "--keyint 60", // "--min-keyint 60", // "--no-scenecut" //} //TEST 3 arguments : new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--pass 1", $"--vbitrate {(_.VideoBitrate)}", $"--gopsize {_.GOPSize}", "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut" } ); if (output.Success && _.StatsFile.Exists) { stdout(output.Output); } else { stderr(output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.Pass1 }); if (!queueOutput.Success) { stderr(queueOutput.Output); } return; } stdout($"*** End PREPROCESSOR1 {DateTime.Now.ToString()}"); #endregion #region Pass 2 stdout($"***Start PREPROCESSOR2 {DateTime.Now.ToString()}"); output = await Utils.RunProcessAsync( prefix : "PREPROCESSOR1", additionalEnvironment : new Dictionary <string, string> { { "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" } }, fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor", //Original //arguments: new[] { // $"--infile {_.LocalFile.FullName}", // $"--stats {_.StatsFile.FullName}", // $"--outfile {_.MmrkFile.FullName}", // $"--pass 2", // $"--vbitrate {_.VideoBitrate}", $"--gopsize {_.GOPSize}", // string.IsNullOrEmpty(_.VideoFilter) ? "" : $"--x264encopts --video-filter {_.VideoFilter}", // //"--bframes 2", // //"--b-pyramid none", // //"--b-adapt 0", // "--keyint 60", // "--min-keyint 60", // "--no-scenecut" //} //TEST 2 //arguments: new[] { // $"--infile {_.LocalFile.FullName}", // $"--stats {_.StatsFile.FullName}", // $"--outfile {_.MmrkFile.FullName}", // $"--pass 2", // $"--vbitrate {(_.VideoBitrate/1000)}K", // $"--gopsize {_.GOPSize}", // "--x264encopts", // "--keyint 60", // "--min-keyint 60", // "--no-scenecut" //} //TEST3 arguments : new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {(_.VideoBitrate)}", $"--gopsize {_.GOPSize}", "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut" } ); if (output.Success && _.MmrkFile.Exists) { stdout(output.Output); } else { stderr(output.Output); stdout($"***Start PREPROCESSOR Error Notification Message {DateTime.Now.ToString()}"); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.Pass2 }); stdout($"***End PREPROCESSOR Error Notification Message {DateTime.Now.ToString()}"); if (!queueOutput.Success) { stderr(queueOutput.Output); } return; } stdout($"***End PREPROCESSOR2 {DateTime.Now.ToString()}"); #endregion #region Upload MMRK stdout($"***Start Upload MMRK {DateTime.Now.ToString()}"); output = await _.MmrkFile.UploadToAsync(_.MmmrkURL); if (output.Success) { stdout(output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Finished, JobOutput = output.Output, Stage = PreprocessorStage.UploadMMRK }); if (!queueOutput.Success) { stderr(queueOutput.Output); } } else { stderr(output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = PreprocessorStage.UploadMMRK }); if (!queueOutput.Success) { stderr(queueOutput.Output); } return; } stdout($"***End Upload MMRK {DateTime.Now.ToString()}"); #endregion } else { #region Download MMRK stdout($"***Start Download MMRK MMRK {DateTime.Now.ToString()}"); output = await _.MmmrkURL.DownloadToAsync(_.MmrkFile); if (output.Success) { stdout(output.Output); } else { stderr(output.Output); var queueOutput = await _.Queue.DispatchMessage(new NotificationEmbedder { AssetID = _.Job.AssetID, JobID = _.Job.JobID, FileName = _.Name, Status = JobStatus.Error, JobOutput = output.Output, Stage = EmbedderStage.DownloadMMRK }); if (!queueOutput.Success) { stderr(queueOutput.Output); } return; } stdout($"***End Download MMRK MMRK {DateTime.Now.ToString()}"); #endregion } }
static async Task <int> MainAsync(string[] args) { #region Install licenses if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("LICENSES"))) { stderr(Category.Main, "Could not determine licenses from environment..."); return(-1); } LicenseData.InjectIntoFilesystem(environmentVariable: "LICENSES"); #endregion var tableConnectionString = Environment.GetEnvironmentVariable("LOGGINGTABLE"); if (string.IsNullOrEmpty(tableConnectionString)) { stdout(Program.Category.Main, $"Could not configure table logging, missing environment variable 'LOGGINGTABLE'"); } var table = await TableWrapper.CreateAsync( connectionString : tableConnectionString, preprocessorTableName : "preprocessor", embedderTableName : "embedder"); #region Read Job EmbedderJobDTO job = null; var jobEnvironmentVariable = Environment.GetEnvironmentVariable("JOB"); if (string.IsNullOrEmpty(jobEnvironmentVariable)) { stderr(Category.Main, "Could not retrieve job from 'JOB' environment variable..."); return(-1); } try { string json = null; if (jobEnvironmentVariable.StartsWith("http")) { var ms = new MemoryStream(); using (var client = new HttpClient()) using (var stream = await client.GetStreamAsync(jobEnvironmentVariable)) { await stream.CopyToAsync(ms); } json = Encoding.UTF8.GetString(ms.GetBuffer()); } else { json = Encoding.UTF8.GetString( Convert.FromBase64String(jobEnvironmentVariable)); } // cat juan.json | jq -c -M . | base64 --wrap=0 > juan.base64 job = JsonConvert.DeserializeObject <EmbedderJobDTO>(json); if (job == null) { stderr(Category.Main, "Could not read job description from 'JOB' environment variable"); return(-1); } } catch (Exception ex) { stderr(Category.Main, $"Could not parse the job description: {ex.Message}"); stderr(Category.Main, $"JOB == {jobEnvironmentVariable}"); return(-1); } #endregion var workFolder = Path.Combine("/mnt", job.JobID .Replace(":", "_") .Replace("/", "_")); stdout(Category.Main, $"Changing work directory to {workFolder}"); Directory.CreateDirectory(workFolder); Environment.CurrentDirectory = workFolder; #region MMRK Generation var preprocessorData = PreprocessorJob.DeterminePreprocessorJobs(job); stdout(Category.Main, "Start Preprocessor"); foreach (var pd in preprocessorData) { // run these compute-intensive jobs sequentially await Program.RunPreprocessorAsync(pd, stdout, stderr, table); } stdout(Category.Main, "Finished Preprocessor"); #endregion #region Watermarking Func <int> getNumberOfParallelEmbedderTasks = () => { int result; if (int.TryParse(Environment.GetEnvironmentVariable("PARALLELEMBEDDERS"), out result)) { return(result); } return(5); }; var parallelEmbedderTasks = getNumberOfParallelEmbedderTasks(); stdout(Category.Main, "Start Embedder"); var embedderData = EmbedderJob.DetermineEmbedderJobs(job); await embedderData.ForEachAsync( parallelTasks : parallelEmbedderTasks, task : _ => Program.RunEmbedderAsync(_, stdout, stderr, table)); stdout(Category.Main, "Finished Embedder"); #endregion #region Delete all MMRK files from pod filesystem foreach (var pd in preprocessorData) { Policy .Handle <Exception>() .WaitAndRetry( retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(1)) .Execute(() => { if (pd.MmrkFile.Exists) { pd.MmrkFile.Delete(); } }); } #endregion Environment.CurrentDirectory = "/"; stdout(Category.Main, $"Removing work directory {workFolder}"); Directory.Delete(workFolder, recursive: true); return(0); }