public PreprocessorTableEntity(PreprocessorJob job, string status)
{
this.PartitionKey = job.Job.AssetID;
this.RowKey = $"{job.MmmrkURL.NormalizedURL().ToPartitionKey()}-{job.VideoBitrate}";
this.VideoURL = job.Mp4URL != null?job.Mp4URL.NormalizedURL() : "";
this.MMRKURL = job.MmmrkURL.NormalizedURL();
this.AssetID = job.Job.AssetID;
this.JobID = job.Job.JobID;
this.Status = status;
}
public Task LogPreprocessorAsync(PreprocessorJob job, string status)
{
if (this.preprocessorTable == null)
{
return(Task.CompletedTask);
}
var entity = new PreprocessorTableEntity(job: job, status: status);
return(Policy.Handle <Exception>()
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(1),
onRetry: (ex, ts) => { Console.WriteLine($"While logging to table: {ex.Message}"); })
.ExecuteAsync(() =>
{
return preprocessorTable.InsertOrReplaceAsync <PreprocessorTableEntity>(entity);
}));
}
private static async Task RunPreprocessorAsync(PreprocessorJob _, Action <Category, string> stdout, Action <Category, string> stderr)
{
ExecutionResult output;
if (_.RunPreprocessorAndUploadMMRK)
{
#region Download MP4
stdout(Category.DownloadMP4, $"Start Download MP4 {_.Mp4URL.AbsoluteUri}");
output = await _.Mp4URL.DownloadToAsync(_.LocalFile);
if (output.Success)
{
stdout(Category.DownloadMP4, output.Output);
}
else
{
stderr(Category.DownloadMP4, output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.DownloadMP4
});
if (!queueOutput.Success)
{
stderr(Category.QueueNotifications, queueOutput.Output);
}
return;
}
stdout(Category.DownloadMP4, $"Finished Download MP4 {_.Mp4URL.AbsoluteUri}");
#endregion
#region Pass 1
//--x264encopts --bframes 2 --b-pyramid none --b-adapt 0 --no-scenecut
stdout(Category.PreprocessorStep1, $"Start {_.LocalFile.FullName}");
output = await Utils.RunProcessAsync(
prefix : "PREPROCESSOR1",
additionalEnvironment : new Dictionary <string, string> {
{ "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" },
{ "TMPDIR", Environment.CurrentDirectory }
},
fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
arguments : new[] {
$"--infile {_.LocalFile.FullName}",
$"--stats {_.StatsFile.FullName}",
$"--pass 1",
$"--vbitrate {(_.VideoBitrate)}",
$"--gopsize {_.GOPSize}",
"--x264encopts",
"--keyint 60",
"--min-keyint 60",
"--no-scenecut"
}
);
if (output.Success && _.StatsFile.Exists)
{
stdout(Category.PreprocessorStep1, "SUCCESS");
// stdout(Category.PreprocessorStep1, output.Output);
}
else
{
stderr(Category.PreprocessorStep1, output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.Pass1
});
if (!queueOutput.Success)
{
stderr(Category.QueueNotifications, queueOutput.Output);
}
return;
}
stdout(Category.PreprocessorStep1, $"Finished {_.LocalFile.FullName}");
#endregion
#region Pass 2
stdout(Category.PreprocessorStep2, $"Start {_.LocalFile.FullName}");
output = await Utils.RunProcessAsync(
prefix : "PREPROCESSOR2",
additionalEnvironment : new Dictionary <string, string> {
{ "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" },
{ "TMPDIR", Environment.CurrentDirectory }
},
fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
//Original arguments: new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {_.VideoBitrate}", $"--gopsize {_.GOPSize}", string.IsNullOrEmpty(_.VideoFilter) ? "" : $"--x264encopts --video-filter {_.VideoFilter}", "--bframes 2", "--b-pyramid none", "--b-adapt 0", "--keyint 60", "--min-keyint 60", "--no-scenecut" }
//TEST 2 arguments: new[] { $"--infile {_.LocalFile.FullName}", $"--stats {_.StatsFile.FullName}", $"--outfile {_.MmrkFile.FullName}", $"--pass 2", $"--vbitrate {(_.VideoBitrate/1000)}K", $"--gopsize {_.GOPSize}", "--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut" }
//TEST3
arguments : new[] {
$"--infile {_.LocalFile.FullName}",
$"--stats {_.StatsFile.FullName}",
$"--outfile {_.MmrkFile.FullName}",
$"--pass 2",
$"--vbitrate {(_.VideoBitrate)}",
$"--gopsize {_.GOPSize}",
"--x264encopts", "--keyint 60", "--min-keyint 60", "--no-scenecut"
}
);
if (output.Success && _.MmrkFile.Exists)
{
stdout(Category.PreprocessorStep2, "SUCCESS");
// stdout(Category.PreprocessorStep2, output.Output);
}
else
{
stderr(Category.PreprocessorStep2, output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.Pass2
});
if (!queueOutput.Success)
{
stderr(Category.QueueNotifications, queueOutput.Output);
}
return;
}
stdout(Category.PreprocessorStep2, $"Finished {_.LocalFile.FullName}");
#endregion
#region Delete MP4 and statistics files
// Delete the input MP4 after MMRK is generated
if (_.LocalFile.Exists)
{
stdout(Category.Main, $"Deleting {_.LocalFile.FullName}");
_.LocalFile.Delete();
}
if (_.StatsFile.Exists)
{
_.StatsFile.Delete();
}
if (File.Exists($"{_.StatsFile.FullName}.mbtree"))
{
File.Delete($"{_.StatsFile.FullName}.mbtree");
}
#endregion
#region Upload MMRK
stdout(Category.UploadMMRK, $"Start Upload MMRK {_.MmrkFile.FullName}");
output = await _.MmrkFile.UploadToAsync(_.MmmrkURL);
if (output.Success)
{
stdout(Category.UploadMMRK, output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Finished,
JobOutput = output.Output,
Stage = PreprocessorStage.UploadMMRK
});
if (!queueOutput.Success)
{
stderr(Category.QueueNotifications, queueOutput.Output);
}
}
else
{
stderr(Category.UploadMMRK, output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.UploadMMRK
});
if (!queueOutput.Success)
{
stderr(Category.QueueNotifications, queueOutput.Output);
}
return;
}
stdout(Category.UploadMMRK, $"End Upload MMRK {_.MmrkFile.FullName}");
#endregion
}
else
{
#region Download MMRK
stdout(Category.DownloadMMRK, $"Start Download MMRK {_.MmrkFile.FullName}");
output = await _.MmmrkURL.DownloadToAsync(_.MmrkFile);
if (output.Success)
{
stdout(Category.DownloadMMRK, output.Output);
}
else
{
stderr(Category.DownloadMMRK, output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationEmbedder
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = EmbedderStage.DownloadMMRK
});
if (!queueOutput.Success)
{
stderr(Category.QueueNotifications, queueOutput.Output);
}
return;
}
stdout(Category.DownloadMMRK, $"Finished Download MMRK {_.MmrkFile.FullName}");
#endregion
}
}
private static async Task RunPreprocessorAsync(PreprocessorJob _, Action <string> stdout, Action <string> stderr)
{
ExecutionResult output;
if (_.RunPreprocessorAndUploadMMRK)
{
#region Download MP4
stdout($"***Start Download MP4 {DateTime.Now.ToString()}");
output = await _.Mp4URL.DownloadToAsync(_.LocalFile);
if (output.Success)
{
stdout(output.Output);
}
else
{
stderr(output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.DownloadMP4
});
if (!queueOutput.Success)
{
stderr(queueOutput.Output);
}
return;
}
stdout($"*** End Download MP4 {DateTime.Now.ToString()}");
#endregion
#region Pass 1
//--x264encopts --bframes 2 --b-pyramid none --b-adapt 0 --no-scenecut
stdout($"*** Start PREPROCESSOR1 {DateTime.Now.ToString()}");
output = await Utils.RunProcessAsync(
prefix : "PREPROCESSOR1",
additionalEnvironment : new Dictionary <string, string> {
{ "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" }
},
fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
//Original
//arguments: new[] {
// $"--infile {_.LocalFile.FullName}",
// $"--stats {_.StatsFile.FullName}",
// $"--pass 1",
// $"--vbitrate {_.VideoBitrate}",
// $"--gopsize {_.GOPSize}",
// "--x264encopts",
// //"--bframes 2",
// //"--b-pyramid none",
// //"--b-adapt 0",
// "--keyint 60",
// "--min-keyint 60",
// "--no-scenecut"
//}
//TEST 2
//arguments: new[] {
// $"--infile {_.LocalFile.FullName}",
// $"--stats {_.StatsFile.FullName}",
// $"--pass 1",
// $"--vbitrate {(_.VideoBitrate/1000)}K",
// $"--gopsize {_.GOPSize}",
// "--x264encopts",
// "--keyint 60",
// "--min-keyint 60",
// "--no-scenecut"
//}
//TEST 3
arguments : new[] {
$"--infile {_.LocalFile.FullName}",
$"--stats {_.StatsFile.FullName}",
$"--pass 1",
$"--vbitrate {(_.VideoBitrate)}",
$"--gopsize {_.GOPSize}",
"--x264encopts",
"--keyint 60",
"--min-keyint 60",
"--no-scenecut"
}
);
if (output.Success && _.StatsFile.Exists)
{
stdout(output.Output);
}
else
{
stderr(output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.Pass1
});
if (!queueOutput.Success)
{
stderr(queueOutput.Output);
}
return;
}
stdout($"*** End PREPROCESSOR1 {DateTime.Now.ToString()}");
#endregion
#region Pass 2
stdout($"***Start PREPROCESSOR2 {DateTime.Now.ToString()}");
output = await Utils.RunProcessAsync(
prefix : "PREPROCESSOR1",
additionalEnvironment : new Dictionary <string, string> {
{ "LD_LIBRARY_PATH", "/usr/share/nexguardescreener-preprocessor/" }
},
fileName : "/usr/share/nexguardescreener-preprocessor/NGS_Preprocessor",
//Original
//arguments: new[] {
// $"--infile {_.LocalFile.FullName}",
// $"--stats {_.StatsFile.FullName}",
// $"--outfile {_.MmrkFile.FullName}",
// $"--pass 2",
// $"--vbitrate {_.VideoBitrate}", $"--gopsize {_.GOPSize}",
// string.IsNullOrEmpty(_.VideoFilter) ? "" : $"--x264encopts --video-filter {_.VideoFilter}",
// //"--bframes 2",
// //"--b-pyramid none",
// //"--b-adapt 0",
// "--keyint 60",
// "--min-keyint 60",
// "--no-scenecut"
//}
//TEST 2
//arguments: new[] {
// $"--infile {_.LocalFile.FullName}",
// $"--stats {_.StatsFile.FullName}",
// $"--outfile {_.MmrkFile.FullName}",
// $"--pass 2",
// $"--vbitrate {(_.VideoBitrate/1000)}K",
// $"--gopsize {_.GOPSize}",
// "--x264encopts",
// "--keyint 60",
// "--min-keyint 60",
// "--no-scenecut"
//}
//TEST3
arguments : new[] {
$"--infile {_.LocalFile.FullName}",
$"--stats {_.StatsFile.FullName}",
$"--outfile {_.MmrkFile.FullName}",
$"--pass 2",
$"--vbitrate {(_.VideoBitrate)}",
$"--gopsize {_.GOPSize}",
"--x264encopts",
"--keyint 60",
"--min-keyint 60",
"--no-scenecut"
}
);
if (output.Success && _.MmrkFile.Exists)
{
stdout(output.Output);
}
else
{
stderr(output.Output);
stdout($"***Start PREPROCESSOR Error Notification Message {DateTime.Now.ToString()}");
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.Pass2
});
stdout($"***End PREPROCESSOR Error Notification Message {DateTime.Now.ToString()}");
if (!queueOutput.Success)
{
stderr(queueOutput.Output);
}
return;
}
stdout($"***End PREPROCESSOR2 {DateTime.Now.ToString()}");
#endregion
#region Upload MMRK
stdout($"***Start Upload MMRK {DateTime.Now.ToString()}");
output = await _.MmrkFile.UploadToAsync(_.MmmrkURL);
if (output.Success)
{
stdout(output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Finished,
JobOutput = output.Output,
Stage = PreprocessorStage.UploadMMRK
});
if (!queueOutput.Success)
{
stderr(queueOutput.Output);
}
}
else
{
stderr(output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationPreprocessor
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = PreprocessorStage.UploadMMRK
});
if (!queueOutput.Success)
{
stderr(queueOutput.Output);
}
return;
}
stdout($"***End Upload MMRK {DateTime.Now.ToString()}");
#endregion
}
else
{
#region Download MMRK
stdout($"***Start Download MMRK MMRK {DateTime.Now.ToString()}");
output = await _.MmmrkURL.DownloadToAsync(_.MmrkFile);
if (output.Success)
{
stdout(output.Output);
}
else
{
stderr(output.Output);
var queueOutput = await _.Queue.DispatchMessage(new NotificationEmbedder
{
AssetID = _.Job.AssetID,
JobID = _.Job.JobID,
FileName = _.Name,
Status = JobStatus.Error,
JobOutput = output.Output,
Stage = EmbedderStage.DownloadMMRK
});
if (!queueOutput.Success)
{
stderr(queueOutput.Output);
}
return;
}
stdout($"***End Download MMRK MMRK {DateTime.Now.ToString()}");
#endregion
}
}
static async Task <int> MainAsync(string[] args)
{
#region Install licenses
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("LICENSES")))
{
stderr(Category.Main, "Could not determine licenses from environment...");
return(-1);
}
LicenseData.InjectIntoFilesystem(environmentVariable: "LICENSES");
#endregion
var tableConnectionString = Environment.GetEnvironmentVariable("LOGGINGTABLE");
if (string.IsNullOrEmpty(tableConnectionString))
{
stdout(Program.Category.Main, $"Could not configure table logging, missing environment variable 'LOGGINGTABLE'");
}
var table = await TableWrapper.CreateAsync(
connectionString : tableConnectionString,
preprocessorTableName : "preprocessor",
embedderTableName : "embedder");
#region Read Job
EmbedderJobDTO job = null;
var jobEnvironmentVariable = Environment.GetEnvironmentVariable("JOB");
if (string.IsNullOrEmpty(jobEnvironmentVariable))
{
stderr(Category.Main, "Could not retrieve job from 'JOB' environment variable...");
return(-1);
}
try
{
string json = null;
if (jobEnvironmentVariable.StartsWith("http"))
{
var ms = new MemoryStream();
using (var client = new HttpClient())
using (var stream = await client.GetStreamAsync(jobEnvironmentVariable))
{
await stream.CopyToAsync(ms);
}
json = Encoding.UTF8.GetString(ms.GetBuffer());
}
else
{
json = Encoding.UTF8.GetString(
Convert.FromBase64String(jobEnvironmentVariable));
}
// cat juan.json | jq -c -M . | base64 --wrap=0 > juan.base64
job = JsonConvert.DeserializeObject <EmbedderJobDTO>(json);
if (job == null)
{
stderr(Category.Main, "Could not read job description from 'JOB' environment variable");
return(-1);
}
}
catch (Exception ex)
{
stderr(Category.Main, $"Could not parse the job description: {ex.Message}");
stderr(Category.Main, $"JOB == {jobEnvironmentVariable}");
return(-1);
}
#endregion
var workFolder =
Path.Combine("/mnt", job.JobID
.Replace(":", "_")
.Replace("/", "_"));
stdout(Category.Main, $"Changing work directory to {workFolder}");
Directory.CreateDirectory(workFolder);
Environment.CurrentDirectory = workFolder;
#region MMRK Generation
var preprocessorData = PreprocessorJob.DeterminePreprocessorJobs(job);
stdout(Category.Main, "Start Preprocessor");
foreach (var pd in preprocessorData)
{
// run these compute-intensive jobs sequentially
await Program.RunPreprocessorAsync(pd, stdout, stderr, table);
}
stdout(Category.Main, "Finished Preprocessor");
#endregion
#region Watermarking
Func <int> getNumberOfParallelEmbedderTasks = () =>
{
int result;
if (int.TryParse(Environment.GetEnvironmentVariable("PARALLELEMBEDDERS"), out result))
{
return(result);
}
return(5);
};
var parallelEmbedderTasks = getNumberOfParallelEmbedderTasks();
stdout(Category.Main, "Start Embedder");
var embedderData = EmbedderJob.DetermineEmbedderJobs(job);
await embedderData.ForEachAsync(
parallelTasks : parallelEmbedderTasks,
task : _ => Program.RunEmbedderAsync(_, stdout, stderr, table));
stdout(Category.Main, "Finished Embedder");
#endregion
#region Delete all MMRK files from pod filesystem
foreach (var pd in preprocessorData)
{
Policy
.Handle <Exception>()
.WaitAndRetry(
retryCount: 5,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(1))
.Execute(() =>
{
if (pd.MmrkFile.Exists)
{
pd.MmrkFile.Delete();
}
});
}
#endregion
Environment.CurrentDirectory = "/";
stdout(Category.Main, $"Removing work directory {workFolder}");
Directory.Delete(workFolder, recursive: true);
return(0);
}