private static IEnumerable <EmbedderJob> DetermineEmbedderJobs(EmbedderJobDTO job)
{
var embedderQueue = new CloudQueue(job.EmbedderNotificationQueue.AsUri());
var ej = job.EmbedderJobs.SelectMany(
_ => _.EmbedderItems,
(a, b) => new EmbedderJob
{
Job = job,
Name = b.FileName,
UserID = a.UserID,
MmrkURL = job.PreprocessorItems.FirstOrDefault(_ => _.FileName == b.FileName)?.MmrkUrl.AsUri(),
MmrkFile = b.FileName.AsMmrkFile(),
WatermarkedFile = b.FileName.AsWatermarkFileForUser(a.UserID),
WatermarkedURL = b.WaterMarkedMp4.AsUri(),
Queue = embedderQueue
})
.Where(_ => _.MmrkURL != null);
return(ej);
}
private static IEnumerable <PreprocessorJob> DeterminePreprocessorJobs(EmbedderJobDTO job)
{
var preprocessorQueue = new CloudQueue(job.PreprocessorNotificationQueue.AsUri());
var pj = job.PreprocessorItems.Select(_ => new PreprocessorJob
{
Job = job,
Name = _.FileName,
LocalFile = _.FileName.AsLocalFile(),
StatsFile = _.FileName.AsStatsFile(),
MmrkFile = _.FileName.AsMmrkFile(),
Mp4URL = _.VideoURL.AsUri(),
MmmrkURL = _.MmrkUrl.AsUri(),
GOPSize = _.GOPSize,
VideoBitrate = _.VideoBitrate,
VideoFilter = _.VideoFilter,
RunPreprocessorAndUploadMMRK = !string.IsNullOrEmpty(_.VideoURL),
Queue = preprocessorQueue
});
return(pj);
}
static async Task <int> MainAsync(string[] args)
{
#region Install licenses
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("LICENSES")))
{
stderr(Category.Main, "Could not determine licenses from environment...");
return(-1);
}
LicenseData.InjectIntoFilesystem(environmentVariable: "LICENSES");
#endregion
#region Read Job
EmbedderJobDTO job = null;
var jobEnvironmentVariable = Environment.GetEnvironmentVariable("JOB");
if (string.IsNullOrEmpty(jobEnvironmentVariable))
{
stderr(Category.Main, "Could not retrieve job from 'JOB' environment variable...");
return(-1);
}
try
{
string json = null;
if (jobEnvironmentVariable.StartsWith("http"))
{
var ms = new MemoryStream();
using (var client = new HttpClient())
using (var stream = await client.GetStreamAsync(jobEnvironmentVariable))
{
await stream.CopyToAsync(ms);
}
json = Encoding.UTF8.GetString(ms.GetBuffer());
}
else
{
json = Encoding.UTF8.GetString(
Convert.FromBase64String(jobEnvironmentVariable));
}
// cat juan.json | jq -c -M . | base64 --wrap=0 > juan.base64
job = JsonConvert.DeserializeObject <EmbedderJobDTO>(json);
if (job == null)
{
stderr(Category.Main, "Could not read job description from 'JOB' environment variable");
return(-1);
}
}
catch (Exception ex)
{
stderr(Category.Main, $"Could not parse the job description: {ex.Message}");
stderr(Category.Main, $"JOB == {jobEnvironmentVariable}");
return(-1);
}
#endregion
var workFolder =
Path.Combine("/mnt", job.JobID
.Replace(":", "_")
.Replace("/", "_"));
stdout(Category.Main, $"Changing work directory to {workFolder}");
Directory.CreateDirectory(workFolder);
Environment.CurrentDirectory = workFolder;
#region MMRK Generation
var preprocessorData = Program.DeterminePreprocessorJobs(job);
stdout(Category.Main, "Start Preprocessor");
foreach (var pd in preprocessorData)
{
// run these compute-intensive jobs sequentially
await Program.RunPreprocessorAsync(pd, stdout, stderr);
}
stdout(Category.Main, "Finished Preprocessor");
#endregion
#region Watermarking
Func <int> getNumberOfParallelEmbedderTasks = () =>
{
int result;
if (int.TryParse(Environment.GetEnvironmentVariable("PARALLELEMBEDDERS"), out result))
{
return(result);
}
return(5);
};
var parallelEmbedderTasks = getNumberOfParallelEmbedderTasks();
stdout(Category.Main, "Start Embedder");
var embedderData = Program.DetermineEmbedderJobs(job);
await embedderData.ForEachAsync(
parallelTasks : parallelEmbedderTasks,
task : _ => Program.RunEmbedderAsync(_, stdout, stderr));
stdout(Category.Main, "Finished Embedder");
#endregion
#region Delete all MMRK files from pod filesystem
foreach (var pd in preprocessorData)
{
Policy
.Handle <Exception>()
.WaitAndRetry(
retryCount: 5,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(1))
.Execute(() =>
{
if (pd.MmrkFile.Exists)
{
pd.MmrkFile.Delete();
}
});
}
#endregion
Environment.CurrentDirectory = "/";
stdout(Category.Main, $"Removing work directory {workFolder}");
Directory.Delete(workFolder, recursive: true);
return(0);
}
static async Task <int> MainAsync(string[] args)
{
Action <string> stdout = Console.Out.WriteLine;
Action <string> stderr = Console.Error.WriteLine;
#region Install licenses
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("LICENSES")))
{
stderr("Could not determine licenses from environment...");
return(-1);
}
LicenseData.InjectIntoFilesystem(environmentVariable: "LICENSES");
#endregion
#region Read Job
EmbedderJobDTO job = null;
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JOB")))
{
stderr("Could not retrieve job from 'JOB' environment variable...");
return(-1);
}
try
{
// cat juan.json | jq -c -M . | base64 --wrap=0 > juan.base64
var environmentVariable = Environment.GetEnvironmentVariable("JOB");
var json = Encoding.UTF8.GetString(
Convert.FromBase64String(environmentVariable));
job = JsonConvert.DeserializeObject <EmbedderJobDTO>(json);
}
catch (Exception ex)
{
stderr(ex.Message);
return(-1);
}
if (job == null)
{
stderr("Could not read job description from 'JOB' environment variable");
return(-1);
}
#endregion
var preprocessorData = Program.DeterminePreprocessorJobs(job);
foreach (var pd in preprocessorData)
{
// run these compute-intensive jobs sequentially
stdout($"***Start Preprocessor {DateTime.Now.ToString()}");
await Program.RunPreprocessorAsync(pd, stdout, stderr);
stdout($"***Finish Preprocessor {DateTime.Now.ToString()}");
}
stdout($"****Start Embedder {DateTime.Now.ToString()}");
var embedderData = Program.DetermineEmbedderJobs(job);
var embedderTasks = embedderData.Select(_ => Program.RunEmbedderAsync(_, stdout, stderr));
await Task.WhenAll(embedderTasks);
stdout($"****Finish Embedder {DateTime.Now.ToString()}");
#region Delete all video files from pod filesystem
foreach (var pd in preprocessorData)
{
Policy
.Handle <Exception>()
.WaitAndRetry(
retryCount: 5,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(1))
.Execute(() =>
{
if (pd.LocalFile.Exists)
{
pd.LocalFile.Delete();
}
if (pd.MmrkFile.Exists)
{
pd.MmrkFile.Delete();
}
});
}
foreach (var ed in embedderData)
{
Policy
.Handle <Exception>()
.WaitAndRetry(
retryCount: 5,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(1))
.Execute(() =>
{
if (ed.WatermarkedFile.Exists)
{
ed.WatermarkedFile.Delete();
}
});
}
#endregion
return(0);
}
