public void DoNotifyWhenBuildIsCompleteTest1()
{
JenkinsJob currentJob = new JenkinsJob("name", "url", false, 0)
{
JobStatus = "FAILED",
LastComletedBuildNumber = 249,
Building = true,
NotifySettings = new NotifySettings()
{
NotifyWhenBuildIsComplete = true
}
};
JenkinsWorkflow NewStatus = new WorkflowJob()
{
lastBuild = new WorkflowRun()
{
building = true, number = 251, result = null
},
lastCompletedBuild = new WorkflowRun()
{
building = false, number = 250, result = "FAILED"
},
};
bool expected = true;
bool actual = currentJob.DoNotify(NewStatus);
Assert.AreEqual(expected, actual);
}
public void Run_CreateOrUpdateRecurringJob_ReturnsNotNullWithWorkflow_WhenExists(string schema)
{
var queryService = new QueryService(new SqlServerStorageOptions()
{
SchemaName = schema
});
UseConnection((conn) =>
{
var job1 = WorkflowJob.MakeJob(() => Run_CreateOrUpdateRecurringJob_ReturnsNotNullWithWorkflow_WhenExists(""), "dflt2", Guid.NewGuid(), 4, "new");
var job2 = WorkflowJob.MakeJob(() => Run_CreateOrUpdateRecurringJob_ReturnsNotNullWithWorkflow_WhenExists("asd"), "dfldasdt2", Guid.NewGuid(), 2, "new2");
job1.ContinueWith(job2);
var sc = new Core.Common.ScheduledTask()
{
Cron = "0 0 0 0",
Workflow = new Workflow(job1),
Name = "recurring_name",
OnlyIfLastFinishedOrFailed = true,
LastInvocation = DateTime.Today,
NextInvocation = DateTime.Today
};
var result = queryService.CreateOrUpdateRecurringJob(sc, conn);
Assert.Equal(1, result);
var recurring = conn.Query <ScheduleEntity>($"SELECT * FROM [{schema}].Schedule").FirstOrDefault();
Assert.Equal(sc.Cron, recurring.Cron);
Assert.Equal(sc.OnlyIfLastFinishedOrFailed, recurring.OnlyIfLastFinishedOrFailed);
Assert.Equal(sc.LastInvocation, recurring.LastInvocation);
Assert.Equal(sc.Name, recurring.Name);
Assert.Equal(sc.NextInvocation, recurring.NextInvocation);
Assert.Equal(JobHelper.ToJson(sc.Workflow), recurring.WorkflowInvocationData);
Assert.Equal(JobHelper.ToJson(ScheduledJobInvocationData.FromScheduledJob(sc)), recurring.JobInvocationData);
});
}
public void CreateWorkflow2()
{
var dl = WorkflowJob.MakeJob(() => WorkflowMethod("Download"));
var co1 = WorkflowJob.MakeJob(() => FailingWorkflowMethod("Do something 1"), maxRetries: 1);
var co2 = WorkflowJob.MakeJob(() => WorkflowMethod("Do something 2"));
var co3 = WorkflowJob.MakeJob(() => WorkflowMethod("Do something 3"));
var se1 = WorkflowJob.MakeJob(() => WorkflowMethod("Send email 1"));
var se3 = WorkflowJob.MakeJob(() => WorkflowMethod("Send email 3"));
var de = WorkflowJob.MakeJob(() => WorkflowMethod("Delete email"));
de.WaitFor(se1, co2, se3);
co1.ContinueWith(se1);
co3.ContinueWith(se3);
var group = WorkflowJobGroup.RunInParallel(co1, co2, co3);
dl.ContinueWithGroup(group);
var wf = new Workflow(dl);
GlobalConfiguration.Configuration.UseSqlServerStorage("db");
var client = new BackgroundJobClient();
client.CreateAndEnqueue(wf);
}
public static WorkflowJob GetWorkflowJob(JobId id = null, WhenFailure ifFailure = WhenFailure.ContinueOn, ShareContext share = ShareContext.Parent, int noOfJobs = 0, JobStatus status = JobStatus.Completed, bool longProcess = false)
{
if (id == null)
{
id = GetJobId();
}
var context = new Mock <IJobContext>();
context.Setup(c => c.ParentJobId).Returns(id);
var rtn = new WorkflowJob(id, context.Object, ifFailure, share);
for (int i = 0; i < noOfJobs; i++)
{
rtn.AddJob(
longProcess?
GetLongRunningJob(
TimeSpan.FromMilliseconds(500),
null, status)
.Object :
GetFakeJob(null, status)
.Object);
}
return(rtn);
}
private async Task <WorkflowJob> CreateWorkflowJobAsync(string inputURL, string jobName, RequestCloudPortEncodeCreateDTO cloudPortEncodeCreateDTO)
{
var workflow = await GetWorkflowByNameAsync(cloudPortEncodeCreateDTO.WorkflowName).ConfigureAwait(false);
var store = await _telestreamCloudStorageProvider.GetStoreByNameAsync(new Uri(cloudPortEncodeCreateDTO.OutputContainer)).ConfigureAwait(false);
var workflowJob = new WorkflowJob
{
Name = jobName
};
// We need to pass all source files
var workflowJobSources = new Dictionary <string, string>();
foreach (var source in workflow.Input.Sources)
{
workflowJobSources[source.Key] = inputURL;
}
var variables = ProcessWorkflowVariables(workflow, cloudPortEncodeCreateDTO);
workflowJob.Inputs = new WorkflowJobInputs(sources: workflowJobSources, variables: variables);
// Configure the encode payload for OpContext and other data that needs to be tunneled thru the payload.
// Pass the OutputContainer in the Payload, so we can get it back from CloudPort.
var payload = new CloudPortPayload()
{
OperationContext = cloudPortEncodeCreateDTO.OperationContext,
OutputContainer = cloudPortEncodeCreateDTO.OutputContainer
};
workflowJob.Payload = JsonConvert.SerializeObject(payload);
// The Storage Reference section of the Input is where the workflow will advertise which Actions (denoted by the Identifier associated with those actions) which have been configured to emit outputs.
// Additionally, this information will indicate which outputs are TRANSIENT (or temporary) and which outputs are PERMANENT (or intended to exist beyond the lifetime of the job)
// The intent of the Storage Reference section is to provide the entity which is submitting to this workflow the requisite information necessary to specify the desired STORAGE that the action should utilize when running.
workflowJob.StorageReferences = new Dictionary <string, WorkflowJobStorageReferences>();
foreach (var workflowStorageReference in workflow.Input.StorageReferences)
{
workflowJob.StorageReferences[workflowStorageReference.Key] = new WorkflowJobStorageReferences(
storeId: store.Id,
folderOffset: ":id"); // use the job id as a blob name prefix.
}
try
{
var job = await _telestreamCloudClientProvider.CloudPortApi.CreateWorkflowJobAsync(workflow.Id, workflowJob).ConfigureAwait(false);
return(job);
}
catch (ApiException ae)
{
throw new GridwichCloudPortApiException("Error calling CreateWorkflowJobAsync.", cloudPortEncodeCreateDTO.OperationContext, ae);
}
}
public async Task <JToken> Handler(JToken @event, ILambdaContext context)
{
var resourceManager = AwsEnvironment.GetAwsV4ResourceManager();
try
{
var jobData = new JobBase
{
Status = "RUNNING",
Progress = 90
};
await resourceManager.SendNotificationAsync(jobData, @event["notificationEndpoint"].ToMcmaObject <NotificationEndpoint>());
}
catch (Exception error)
{
Logger.Error("Failed to send notification: {0}", error);
}
var jobProfiles = await resourceManager.GetAsync <JobProfile>(("name", "AiWorkflow"));
var jobProfileId = jobProfiles?.FirstOrDefault()?.Id;
if (jobProfileId == null)
{
throw new Exception("JobProfile 'AiWorkflow' not found");
}
var workflowJob = new WorkflowJob
{
JobProfile = jobProfileId,
JobInput = new JobParameterBag
{
["bmContent"] = @event["data"]["bmContent"],
["bmEssence"] = @event["data"]["bmEssence"]
}
};
workflowJob = await resourceManager.CreateAsync(workflowJob);
return(JToken.FromObject(new
{
aiWorkflow = workflowJob.Id,
bmContent = @event["data"]["bmContent"],
websiteMediaFile = @event["data"]["websiteFile"]
}));
}
private static List <string> GetOutputFiles(WorkflowJob workflowJob)
{
var listOfOutputFiles = new List <string>();
foreach (var job in workflowJob.ActionJobs.OrEmptyIfNull())
{
foreach (var encodeOutput in job.Outputs.OrEmptyIfNull())
{
foreach (var outputFile in encodeOutput.Files.OrEmptyIfNull())
{
var blobName = encodeOutput.RemotePath.TrimEnd('/') + "/" + outputFile;
listOfOutputFiles.Add(blobName);
}
}
}
return(listOfOutputFiles);
}
public void Workflow()
{
var rootJob = WorkflowJob.MakeJob(() => RecurringMethod("1 task", 1));
rootJob.ContinueWith(WorkflowJob.MakeJob(() => RecurringMethod("2 task", 2)))
.ContinueWithGroup(
WorkflowJobGroup.RunInParallel(
WorkflowJob.MakeJob(() => RecurringMethod("3 task", 3)),
WorkflowJob.MakeJob(() => RecurringMethod("4 task", 4))
)).ContinueWith(
WorkflowJob.MakeJob(() => RecurringMethod("5 task", 5)
));
var wf = new Workflow(rootJob);
wf.SaveWorkflow((t) => { t.QueueJob.JobId = (int)t.QueueJob.Job.Arguments[1]; Debug.WriteLine($"SAved: {t.QueueJob.JobId} - {t.QueueJob.Job.Arguments[0]}"); return((int)t.QueueJob.Job.Arguments[1]); });
var jobs = wf.GetAllJobs().ToList();
}
public static WorkflowJob GetNestedWorkflowWithRandomStatus(TimeSpan timeToRun, int noOfJobs = 2)
{
JobId id = GetJobId();
var context = new Mock <IJobContext>();
context.Setup(c => c.ParentJobId).Returns(id);
var rtn = new WorkflowJob(id, context.Object, WhenFailure.ContinueOn, ShareContext.Parent);
var rn = new Random();
for (int i = 0; i < noOfJobs; i++)
{
var n = rn.Next(10);
n = n == 0 ? 2 : n;
rtn.AddJob(GetWorkflowJob(noOfJobs: n, longProcess: true, status: n % 2 == 0 ? JobStatus.Completed: JobStatus.CompletedWithError));
}
return(rtn);
}
public void CreateRecurringWorkflow()
{
var dl = WorkflowJob.MakeJob(() => WorkflowMethod("Download"));
var co1 = WorkflowJob.MakeJob(() => WorkflowMethod("Do something 1"));
var co2 = WorkflowJob.MakeJob(() => WorkflowMethod("Do something 2"));
var co3 = WorkflowJob.MakeJob(() => WorkflowMethod("Do something 3"));
var se1 = WorkflowJob.MakeJob(() => WorkflowMethod("Send email 1"));
var se3 = WorkflowJob.MakeJob(() => WorkflowMethod("Send email 3"));
var de = WorkflowJob.MakeJob(() => WorkflowMethod("Delete email"));
de.WaitFor(se1, co2, se3);
co1.ContinueWith(se1);
co3.ContinueWith(se3);
var group = WorkflowJobGroup.RunInParallel(co1, co2, co3);
dl.ContinueWithGroup(group);
var wf = new Workflow(dl);
GlobalConfiguration.Configuration.UseSqlServerStorage("db");
RecurringWorkflow.AddOrUpdate("test workflow", wf, Cron.MinuteInterval(1));
}
public bool DoNotify(JenkinsWorkflow NewStatus)
{
if (temporary)
{
WorkflowRun tempJobNewStatus = NewStatus as WorkflowRun;
if ((notifySettings.NotifyWhenJobStateChanges || notifySettings.NotifyWhenBuildIsComplete || notifySettings.NotifyWhenJobBecomesGreen || notifySettings.NotifyWhenJobBecomesRed))
{
if (this.building != tempJobNewStatus.building)
{
return(!tempJobNewStatus.building);
}
}
}
else
{
WorkflowJob jobNewStatus = NewStatus as WorkflowJob;
if (notifySettings.NotifyWhenJobStateChanges && IsLastBuildCompleted(jobNewStatus) && this.status != jobNewStatus.lastCompletedBuild.result)
{
return(true);
}
if (notifySettings.NotifyWhenBuildIsComplete && IsLastBuildCompleted(jobNewStatus))
{
return(true);
}
if (notifySettings.NotifyWhenJobBecomesGreen && this.status != jobNewStatus.lastCompletedBuild.result && jobNewStatus.lastCompletedBuild.result == SUCCESS)
{
return(true);
}
if (notifySettings.NotifyWhenJobBecomesRed && this.status != jobNewStatus.lastCompletedBuild.result && jobNewStatus.lastCompletedBuild.result != SUCCESS)
{
return(true);
}
}
return(false);
}
public void UpdateJobStatus(JenkinsWorkflow NewStatus)
{
if (temporary)
{
WorkflowRun tempJobNewStatus = NewStatus as WorkflowRun;
this.status = tempJobNewStatus.result;
this.building = tempJobNewStatus.building;
this.color = GetColorForStatus(status);
}
else
{
WorkflowJob jobNewStatus = NewStatus as WorkflowJob;
if (jobNewStatus.lastCompletedBuild != null)
{
this.status = jobNewStatus.lastCompletedBuild.result;
this.lastCompletedBuildNumber = jobNewStatus.lastCompletedBuild.number;
}
if (jobNewStatus.lastBuild != null)
{
this.building = jobNewStatus.lastBuild.building;
}
this.color = GetColorForStatus(status);
}
}
public void CreateWorkflow()
{
GlobalConfiguration.Configuration.UseSqlServerStorage("db");
var contextId = Guid.NewGuid();
var job1 = WorkflowJob.MakeJob(() => WorkflowMethod("1 task"), contextId: contextId);
var job2 = WorkflowJob.MakeJob(() => WorkflowMethod("2 task"), contextId: contextId);
var job31 = WorkflowJob.MakeJob(() => WorkflowMethod("3.1 task"), contextId: contextId);
var job32 = WorkflowJob.MakeJob(() => WorkflowMethod("3.2 task"), contextId: contextId);
var job321 = WorkflowJob.MakeJob(() => WorkflowMethod("3.2.1 task"), contextId: contextId);
var group = WorkflowJobGroup.RunInParallel(
job31,
job32
);
var job4 = WorkflowJob.MakeJob(() => WorkflowMethod("4 task"), contextId: contextId);
var job5 = WorkflowJob.MakeJob(() => WorkflowMethod("5 task"), contextId: contextId);
job1.ContinueWith(job2);
job2.ContinueWithGroup(group);
group.ContinueWith(job4);
job4.ContinueWith(job5);
job32.ContinueWith(job321);
//rootJob.ContinueWith(WorkflowJob.MakeJob(() => RecurringMethod("2 task", 2))
// .ContinueWithGroup(
// WorkflowJob.MakeJob(() => RecurringMethod("3 task", 3)),
// WorkflowJob.MakeJob(() => RecurringMethod("4 task", 4))
// ).ContinueWith(WorkflowJob.MakeJob(() => RecurringMethod("5 task", 5))));
var wf = new Workflow(job1);
var client = new BackgroundJobClient();
client.CreateAndEnqueue(wf);
}
public void StartNewJobFromTemplate(JenkinsTemplate Template, string JobName)
{
JenkinsJob newJob = new JenkinsJob(JobName, Template.JobUrl, false, lastIndex + 1);
WorkflowJob job = GetJobReport(newJob) as WorkflowJob;
int lastBuildNumberOfJob = job.lastBuild.number;
string jobInputParams = GetJsonInput(Template.GetInput());
HttpWebResponse response = client.Post(Template.BuildJobUrl, jobInputParams);
string location = response.Headers["Location"];
Threading.Thread adder = new Threading.Thread(() =>
{
int numOfMaxTrial = 20;
int lastNumber = lastBuildNumberOfJob;
do
{
Threading.Thread.Sleep(2000);
WorkflowJob report = GetJobReport(newJob) as WorkflowJob;
if (report != null)
{
lastNumber = report.lastBuild.number;
}
numOfMaxTrial--;
}while (lastBuildNumberOfJob == lastNumber && numOfMaxTrial > 0);
newJob.IsTemporary = true;
newJob.RemoveIfCompleted = true;
newJob.NotifySettings = new NotifySettings()
{
NotifyWhenBuildIsComplete = true
};
newJob.JobUrl = String.Format("{0}/{1}", newJob.JobUrl, lastNumber);
AddNewJob(newJob);
});
adder.Start();
}
private bool IsLastBuildCompleted(WorkflowJob NewStatus)
{
return(this.lastCompletedBuildNumber != NewStatus.lastCompletedBuild.number);
}
public async void CloudPortTestWithGoodAndBadData(RequestCloudPortEncodeCreateDTO encodeCreateData, bool blobShouldExist, Type shouldThrowThis, Uri sasUri)
{
// Arrange
var wfj = new WorkflowJob();
Mock.Get(_telestreamCloudClientProvider.CloudPortApi)
.Setup(x => x.CreateWorkflowJobAsync(It.IsAny <string>(), It.IsAny <WorkflowJob>()))
.ReturnsAsync(wfj);
var wf1 = new Workflow(name: "TestWorkflow2")
{
Input = new WorkflowInput
{
Sources = new Dictionary <string, VantageNickName>()
{
{ "any value", new VantageNickName() }
},
Variables = new Dictionary <string, VantageVariable>(),
}
};
wf1.Input.Variables.Add("var1", new VantageVariable("0", "0"));
var wf2 = new Workflow();
WorkflowsCollection wfc = new WorkflowsCollection()
{
Workflows = new List <Workflow>()
{
wf1, wf2
}
};
Mock.Get(_telestreamCloudClientProvider.CloudPortApi)
.Setup(x => x.ListWorkflowsAsync(null, null, null))
.ReturnsAsync(wfc);
Mock.Get(_telestreamCloudClientProvider.CloudPortApi)
.Setup(x => x.GetWorkflowAsync(It.IsAny <string>()))
.ReturnsAsync(wf1);
var sr = new StorageReference();
wf1.Input.StorageReferences = new Dictionary <string, StorageReference>()
{
{ "DavesStorageReference", sr }
};
Mock.Get(_storageService)
.Setup(x => x.GetBlobExistsAsync(It.IsAny <Uri>(), It.IsAny <StorageClientProviderContext>()))
.ReturnsAsync(blobShouldExist);
Mock.Get(_storageService)
.Setup(x => x.GetSasUrlForBlob(It.IsAny <Uri>(), It.IsAny <TimeSpan>(), It.IsAny <StorageClientProviderContext>()))
.Returns(sasUri?.ToString());
Mock.Get(_settingsProvider)
.Setup(x => x.GetAppSettingsValue(It.IsAny <string>()))
.Returns("telestreamCloudApiKey");
Mock.Get(_telestreamCloudStorageProvider)
.Setup(x => x.GetStoreByNameAsync(It.IsAny <Uri>()))
.ReturnsAsync(new Store());
// Act
var cloudPortService = new CloudPortService(_storageService, _telestreamCloudClientProvider, _telestreamCloudStorageProvider);
var ex = await Record.ExceptionAsync(async() => await cloudPortService.EncodeCreateAsync(encodeCreateData).ConfigureAwait(false)).ConfigureAwait(false);
// Assert
if (shouldThrowThis is null)
{
// if there are no throws, test is successful.
Assert.Null(ex);
}
else
{
Assert.NotNull(ex);
Assert.IsType(shouldThrowThis, ex);
}
}
