public void PlanTwoJobsWithMultipleTranscodingPluginsTest()
{
var wfsPluginMock = CreateWfsMock("1");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var ffmpegPluginMock = CreateFFMpegMock("1");
ffmpegPluginMock.SetupGet(p => p.Busy).Returns(false);
ffmpegPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object, ffmpegPluginMock.Object
}, JobRepository, Logging, _callBackService.Object);
var audioJob = CreateNewFFMpegJob();
var videoJob = CreateNewWfsJob();
JobRepository.Add(audioJob);
JobRepository.Add(videoJob);
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(2));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(0));
planner.Calculate();
ffmpegPluginMock.Verify(x => x.CheckAndEstimate(It.IsAny <ExecutionTask>()), Times.Once(), "ffmpeg plugin should only be called once");
wfsPluginMock.Verify(x => x.CheckAndEstimate(It.IsAny <ExecutionTask>()), Times.Once(), "Wfs plugin should only be called once");
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(2));
var dbAudioJob = JobRepository.ActiveJobs().First(j =>
j.Plan.Tasks.First().PluginUrn == ffmpegPluginMock.Object.Urn);
var dbVideoJob = JobRepository.ActiveJobs().First(j =>
j.Plan.Tasks.First().PluginUrn == wfsPluginMock.Object.Urn);
Assert.That(dbAudioJob.Urn, Is.EqualTo(audioJob.Urn));
Assert.That(dbVideoJob.Urn, Is.EqualTo(videoJob.Urn));
}
public void PlanAJobWithMuxingTest()
{
var wfsPluginMock = CreateWfsMock("1");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var ffmpegPluginMock = CreateFFMpegMock("1");
ffmpegPluginMock.SetupGet(p => p.Busy).Returns(false);
ffmpegPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object, ffmpegPluginMock.Object
}, JobRepository, Logging, _callBackService.Object);
var audioJob = CreateNewAudioMuxJob();
JobRepository.Add(audioJob);
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(1));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(0));
planner.Calculate();
ffmpegPluginMock.Verify(x => x.CheckAndEstimate(It.IsAny <ExecutionTask>()), Times.Exactly(1), "ffmpeg plugin should only be called once for muxing");
wfsPluginMock.Verify(x => x.CheckAndEstimate(It.IsAny <ExecutionTask>()), Times.Exactly(1), "Wfs plugin should only be called once for transcoding");
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(1));
var dbMuxJob = JobRepository.ActiveJobs().First();
Assert.That(dbMuxJob.Plan.Tasks.Count(), Is.EqualTo(2));
Assert.That(dbMuxJob.Plan.Tasks[0].PluginUrn, Is.EqualTo(ffmpegPluginMock.Object.Urn));
Assert.That(dbMuxJob.Plan.Tasks[1].PluginUrn, Is.EqualTo(wfsPluginMock.Object.Urn));
}
public void CalculateDoesNotReserveTheLastFFmpegPluginForHighPriorityOnly()
{
var wfsPluginMock = CreateFFMpegMock("1");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var wfsPluginMock2 = CreateFFMpegMock("2");
wfsPluginMock2.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock2.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object, wfsPluginMock2.Object
}, JobRepository, Logging, _callBackService.Object);
var jobMedium = CreateNewFFMpegJob();
jobMedium.Priority = Priority.medium;
JobRepository.Add(jobMedium);
var jobLow = CreateNewFFMpegJob();
jobLow.Priority = Priority.low;
JobRepository.Add(jobLow);
planner.Calculate();
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(2));
}
public void OnlyHighPriorityJobsWillBeGivenTheLastWfsNode()
{
var wfsPluginMock = CreateWfsMock("1");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object
}, JobRepository, Logging, _callBackService.Object);
var jobMedium = CreateNewWfsJob();
jobMedium.Priority = Priority.medium;
JobRepository.Add(jobMedium);
planner.Calculate();
// Medium prio job will be started with only one wfs node
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(1));
JobRepository.Reset();
jobMedium = CreateNewWfsJob();
jobMedium.Priority = Priority.medium;
JobRepository.Add(jobMedium);
var jobHigh = CreateNewWfsJob();
jobHigh.Priority = Priority.high;
JobRepository.Add(jobHigh);
planner.Calculate();
// High prio job will start with one wfs node
Assert.That(JobRepository.ActiveJobs().First().Urn, Is.EqualTo(jobHigh.Urn));
Assert.That(JobRepository.WaitingJobs().First().Urn, Is.EqualTo(jobMedium.Urn));
JobRepository.ActiveJobs().First().Plan.GetCurrentTask().State = ExecutionState.Done;
JobRepository.ActiveJobs().First().Plan.MoveToNextTask();
JobRepository.Reset();
var wfsPluginMock2 = CreateWfsMock("2");
wfsPluginMock2.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock2.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object, wfsPluginMock2.Object
}, JobRepository, Logging, _callBackService.Object);
var jobMedium2 = CreateNewWfsJob();
jobMedium2.Priority = Priority.medium;
JobRepository.Add(jobMedium2);
planner.Calculate();
// Medium prio job will start with two wfs nodes
Assert.That(JobRepository.ActiveJobs().First().Urn, Is.EqualTo(jobMedium2.Urn));
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
}
public override void Calculate()
{
// TODO: cancel, pause, retry , resume...
var currentPlans = JobRepository.ActiveJobs(); //TODO: check already running task before planning next.
var waiting = JobRepository.WaitingJobs().OrderBy(j => j.DueDate).GetEnumerator();
if (!waiting.MoveNext())
{
return;
}
foreach (var freePlugin in Plugins.Where(p => !p.Busy && p.PluginType == DummyLogoPP.Type))
{
var tcPlugin = Plugins.First(p => p.PluginType == DummyTCOnly.Type);
var et1 = new ExecutionTask
{
From = new Essence(waiting.Current.Source),
To = new Essence(waiting.Current.Source)
{
Flags = waiting.Current.Destination.Flags, Files = null
},
PluginUrn = freePlugin.Urn
};
var et2 = new ExecutionTask
{
From = new Essence(et1.To),
To = new Essence(waiting.Current.Destination),
PluginUrn = tcPlugin.Urn
};
if (!freePlugin.CheckAndEstimate(et1))
{
continue;
}
if (!tcPlugin.CheckAndEstimate(et2))
{
continue;
}
var ep = new ExecutionPlan {
Tasks = new List <ExecutionTask> {
et1, et2
}
};
waiting.Current.Plan = ep;
JobRepository.Update(waiting.Current);
if (!waiting.MoveNext())
{
break;
}
}
}
public void MaxFiveMinutesJobsWillBeTreatedAsHighPriorityJobsAndBeGivenTheLastWfsNode()
{
var wfsPluginMock = CreateWfsMock("1");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var wfsPluginMock2 = CreateWfsMock("2");
wfsPluginMock2.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock2.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object, wfsPluginMock2.Object
}, JobRepository, Logging, _callBackService.Object);
var jobHighPrio = CreateNewWfsJob();
jobHighPrio.Priority = Priority.high;
JobRepository.Add(jobHighPrio);
var jobMediumPrioLong = CreateNewWfsJob();
jobMediumPrioLong.Priority = Priority.medium;
jobMediumPrioLong.Source.Duration = 300001;
JobRepository.Add(jobMediumPrioLong);
planner.Calculate();
// Medium prio long job (more than 5 min) will not start if one node left
Assert.That(JobRepository.WaitingJobs().First().Urn, Is.EqualTo(jobMediumPrioLong.Urn));
JobRepository.Reset();
var jobMediumPrioShort = CreateNewWfsJob();
jobMediumPrioShort.Priority = Priority.medium;
jobMediumPrioShort.Source.Duration = 30000;
JobRepository.Add(jobMediumPrioShort);
JobRepository.Add(jobHighPrio);
planner.Calculate();
// Medium prio short job (less than 5 min) will start if one node left
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
Assert.That(JobRepository.ActiveJobs().Select(t => t.Urn == jobMediumPrioShort.Urn), Is.Not.Null);
}
public void PlanOneJobTest()
{
var wfsPluginMock = CreateWfsMock("1");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object
}, JobRepository, Logging, _callBackService.Object);
var job = CreateNewWfsJob();
var jobUrn = job.Urn;
JobRepository.Add(job);
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(1));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(0));
planner.Calculate();
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(1));
Assert.That(JobRepository.ActiveJobs().First().Urn, Is.EqualTo(jobUrn));
}
private Dictionary <string, PluginStats> GetPluginStats()
{
var pluginTypes = Plugins.Select(p => p.PluginType).Distinct();
// these plugins is already in use or planed to be used in the future, count'em as busy.
var reservedPluginUrns = JobRepository.ActiveJobs()
.SelectMany(j => j.Plan.Tasks.Where(t => t.State != ExecutionState.Done).Select(t => t.PluginUrn))
.Distinct();
return(pluginTypes.ToDictionary(type => type, type => new PluginStats()
{
TotalCount = Plugins.Count(p => p.PluginType == type),
BusyCount = Plugins.Count(
p => p.PluginType == type &&
(p.Busy || (p.AsyncOperation && reservedPluginUrns.Contains(p.Urn)))),
FreePlugin = new Stack <IPlugin>(Plugins.Where(
p => p.PluginType == type &&
!(p.Busy || (p.AsyncOperation && reservedPluginUrns.Contains(p.Urn)))))
}));
}
public void EssenceGetsCleanedOnUpdate()
{
var originJob = ActiveJob();
int essenceAmount;
JobRepository.Add(originJob);
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(1));
Assert.That(JobRepository.DoneJobs(), Is.Empty);
Assert.That(JobRepository.WaitingJobs(), Is.Empty);
var jobFromRepo = JobRepository.Get(originJob.Urn);
jobFromRepo.Plan.GetCurrentTask().State = ExecutionState.Done;
jobFromRepo.Plan.MoveToNextTask();
JobRepository.Update(jobFromRepo);
using (var db = new MarvinEntities())
{
essenceAmount = db.essence.Count();
}
//There should only be four essencefiles (two job essence and two task essence)
Assert.That(essenceAmount, Is.EqualTo(4));
}
public void PlanAnAudioJobWhileMuxingTest()
{
var wfsPluginMock = CreateWfsMock("1");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var ffmpegPluginMock = CreateFFMpegMock("1");
ffmpegPluginMock.SetupGet(p => p.Busy).Returns(false);
ffmpegPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object, ffmpegPluginMock.Object
}, JobRepository, Logging, _callBackService.Object);
//Plan the mux job
var muxJob = CreateNewAudioMuxJob();
JobRepository.Add(muxJob);
planner.Calculate();
//Advance muxjob progress to wfs, leaving the ffmpeg plugin free
var dbMuxJob = JobRepository.ActiveJobs().First();
dbMuxJob.Plan.Tasks[0].State = ExecutionState.Done;
dbMuxJob.Plan.Tasks[1].State = ExecutionState.Running;
JobRepository.Update(dbMuxJob);
//Plan the audio trancoding job
var audioJob = CreateNewFFMpegJob();
JobRepository.Add(audioJob);
planner.Calculate();
//Check both jobs are running
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(0));
Assert.That(JobRepository.ActiveJobs().Count(), Is.EqualTo(2));
}
public override void Calculate()
{
// TODO: cancel, pause, retry , resume...
var currentPlans = JobRepository.ActiveJobs(); //TODO: check already running task before planning next.
var waiting = JobRepository.WaitingJobs().OrderBy(j => j.DueDate).GetEnumerator();
if (!waiting.MoveNext())
{
return;
}
foreach (var freePlugin in Plugins.Where(p => !p.Busy))
{
var et = new ExecutionTask
{
From = new Essence(waiting.Current.Source),
To = new Essence(waiting.Current.Destination),
PluginUrn = freePlugin.Urn
};
if (!freePlugin.CheckAndEstimate(et))
{
continue;
}
var ep = new ExecutionPlan {
Tasks = new List <ExecutionTask> {
et
}
};
waiting.Current.Plan = ep;
JobRepository.Update(waiting.Current);
if (!waiting.MoveNext())
{
break;
}
}
}
public void PrioritySortTest()
{
var jobDue1DayLow = CreateNewWfsJob(dueDate: TimeProvider.GetUtcNow().AddDays(1));
jobDue1DayLow.Priority = Priority.low;
JobRepository.Add(jobDue1DayLow);
var jobDue1DayHigh = CreateNewWfsJob(dueDate: TimeProvider.GetUtcNow().AddDays(1));
jobDue1DayHigh.Priority = Priority.high;
JobRepository.Add(jobDue1DayHigh);
var jobDueNowDayLow = CreateNewWfsJob();
jobDueNowDayLow.Priority = Priority.low;
JobRepository.Add(jobDueNowDayLow);
var jobDueNowDayHigh = CreateNewWfsJob();
jobDueNowDayHigh.Priority = Priority.high;
JobRepository.Add(jobDueNowDayHigh);
Assert.That(JobRepository.WaitingJobs().Count(), Is.EqualTo(4));
var wfsPluginMock = CreateWfsMock("1");
var wfsPluginMock2 = CreateWfsMock("2");
wfsPluginMock.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
wfsPluginMock2.SetupGet(p => p.Busy).Returns(false);
wfsPluginMock2.Setup(p => p.CheckAndEstimate(It.IsAny <ExecutionTask>())).Returns(true);
var planner = new SimplePlanner(new List <IPlugin> {
wfsPluginMock.Object, wfsPluginMock2.Object
}, JobRepository, Logging, _callBackService.Object);
planner.Calculate();
var activeJob = JobRepository.ActiveJobs().First(m => m.Id == jobDueNowDayHigh.Id);
Assert.That(activeJob.Urn, Is.EqualTo(jobDueNowDayHigh.Urn));
activeJob.Plan.GetCurrentTask().State = ExecutionState.Done;
activeJob.Plan.MoveToNextTask();
JobRepository.Update(activeJob);
planner.Calculate();
activeJob = JobRepository.ActiveJobs().First(m => m.Id == jobDue1DayHigh.Id);
Assert.That(activeJob.Urn, Is.EqualTo(jobDue1DayHigh.Urn));
activeJob.Plan.GetCurrentTask().State = ExecutionState.Done;
activeJob.Plan.MoveToNextTask();
JobRepository.Update(activeJob);
planner.Calculate();
activeJob = JobRepository.ActiveJobs().First(m => m.Id == jobDueNowDayLow.Id);
Assert.That(activeJob.Urn, Is.EqualTo(jobDueNowDayLow.Urn));
activeJob.Plan.GetCurrentTask().State = ExecutionState.Done;
activeJob.Plan.MoveToNextTask();
JobRepository.Update(activeJob);
planner.Calculate();
activeJob = JobRepository.ActiveJobs().First(m => m.Id == jobDue1DayLow.Id);
Assert.That(activeJob.Urn, Is.EqualTo(jobDue1DayLow.Urn));
activeJob.Plan.GetCurrentTask().State = ExecutionState.Done;
activeJob.Plan.MoveToNextTask();
JobRepository.Update(activeJob);
Assert.That(JobRepository.WaitingJobs(), Is.Empty);
Assert.That(JobRepository.ActiveJobs(), Is.Empty);
Assert.That(JobRepository.DoneJobs().Count(), Is.EqualTo(4));
}