private void Form1_Load(object sender, EventArgs e)
{
ProcessorManager procMgr = new ProcessorManager();
string basePath = Assembly.GetExecutingAssembly().Location;
string baseFolder = Path.GetDirectoryName(basePath);
string procPaths = Path.Combine(baseFolder, "app_files\\processors");
var lstProc = procMgr.GetProcessors(procPaths);
comboBox1.DataSource = lstProc;
comboBox1.DisplayMember = "Name";
//comboBox1.ValueMember = "Processor";
}
public static XmlElement Initialize(XmlElement pipelinesNode)
{
Log.Debug("Pipelines RAW configuration: {0}", pipelinesNode.OuterXml);
Definitions.Clear();
var resultXmlConfig = XmlDocumentEx.LoadXml("<pipelines />");
foreach (XmlElement element in pipelinesNode.ChildNodes.OfType <XmlElement>())
{
string pipelineName = element.Name;
string title = element.GetAttribute("title");
Assert.IsNotNullOrEmpty(title, "The '{0}' pipeline definition doesn't contain the title attribute".FormatWith(pipelineName));
var pipelineNode = resultXmlConfig.DocumentElement.AddElement(pipelineName);
pipelineNode.SetAttribute("title", title);
List <StepDefinition> steps = new List <StepDefinition>();
XmlNodeList stepNodes = element.SelectNodes("step");
// Processors are placed into steps
if (stepNodes != null && stepNodes.Count > 0)
{
foreach (XmlElement step in stepNodes.OfType <XmlElement>())
{
// Clever mechanism of steps reusing. Doesn't seem to be used somewhere.
string fromPipeline = step.GetAttribute("pipeline");
string args = step.GetAttribute("args").EmptyToNull();
if (!string.IsNullOrEmpty(fromPipeline))
{
PipelineDefinition def = Definitions[fromPipeline];
if (args != null)
{
def.Steps.ForEach(s => s.ArgsName = args);
}
AddSteps(steps, def);
}
// All step's processors are interted to the step
else
{
var stepNode = pipelineNode.AddElement("step");
stepNode.SetAttribute("args", args);
List <ProcessorDefinition> processorDefinitions = ProcessorManager.GetProcessors(step, pipelineName, stepNode);
AddStep(steps, processorDefinitions);
}
}
}
// Otherwise all the processors are placed into the default step.
else
{
List <ProcessorDefinition> processorDefinitions = ProcessorManager.GetProcessors(element, pipelineName, pipelineNode);
steps.Add(new StepDefinition(processorDefinitions));
}
Definitions.Add(pipelineName, new PipelineDefinition
{
Steps = steps,
Title = title
});
}
return(resultXmlConfig.DocumentElement);
}
protected override async System.Threading.Tasks.Task ExecuteAsync(CancellationToken stoppingToken)
{
string projectRootPath = _hostingEnvironment.ContentRootPath;
string processorPath = Path.Combine(projectRootPath, "app_files", "processors");
PluginBase.ProcessorManager pm = new ProcessorManager();
var procsFromDisk = pm.GetProcessors(processorPath);
List <string> procsDiskNames = new List <string>();
foreach (ProcessorDTO proc in procsFromDisk)
{
procsDiskNames.Add(proc.Name.ToLower());
}
// Create a new scope to retrieve scoped services
List <string> procsInDb = new List <string>();
using (var scope = _serviceProvider.CreateScope())
{
// Get the DbContext instance
var procService = scope.ServiceProvider.GetRequiredService <IProcessorService>();
//var dbProcs = await procService.GetAll();
var dbProcs = await procService.GetAll();
//var dbProcs = procs.Result;
List <Processor> lstProcs = new List <Processor>();
foreach (Processor proc in dbProcs)
{
proc.enabled = false;
lstProcs.Add(proc);
procsInDb.Add(proc.name.ToLower());
}
//Set all the processors in the db to enabled = false
await procService.Update(lstProcs.ToArray());
var procsIntersect = procsDiskNames.Intersect(procsInDb);
lstProcs = new List <Processor>();
foreach (string proc in procsIntersect)
{
var processor = await procService.GetById(proc);
//Processor processor = result.Result;
processor.enabled = true;
lstProcs.Add(processor);
}
//Set all the processors in the db that are also on disk to enabled = true
await procService.Update(lstProcs.ToArray());
var newProcs = procsDiskNames.Except(procsInDb);
List <ProcessorDTO> lstProcDTO = new List <ProcessorDTO>();
foreach (string proc in newProcs)
{
lstProcDTO.Add(procsFromDisk.Find(x => x.Name.ToLower() == proc.ToLower()));
}
foreach (ProcessorDTO proc in lstProcDTO)
{
Processor processor = new Processor();
processor.enabled = true;
processor.description = proc.Description;
processor.file_type = proc.InstrumentFileType;
processor.id = proc.UniqueId.ToLower();
processor.name = proc.Name.ToLower();
await procService.Create(processor);
}
}
//return System.Threading.Tasks.Task.CompletedTask;
return;
}
public async System.Threading.Tasks.Task RunTask(string id)
{
var task = await _context.Tasks.SingleAsync(t => t.id == id);
Log.Information("Executing Task. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID);
// Step 1: If status!="SCHEDULED" cancel task
if (!task.status.Equals("SCHEDULED"))
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Current Status: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.status, "Task status is not marked as schedulled.");
await this.UpdateStatus(task.id, "CANCELLED", "Task status was set to: " + task.status);
return;
}
// Step 2: Change status to "STARTING"
await this.UpdateStatus(task.id, "STARTING", "");
var workflow = await _context.Workflows.Where(w => w.id == task.workflowID).FirstOrDefaultAsync();
if (workflow == null)
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Unable to find Workflow for the Task.");
await this.UpdateStatus(task.id, "CANCELLED", "Error attempting to get workflow of this task. Workflow ID: " + task.workflowID);
return;
}
// Step 3: Check source directory for files
List <string> files = new List <string>();
string dirFileMessage = "";
if (new DirectoryInfo(@workflow.inputFolder).Exists)
{
files = Directory.GetFiles(@workflow.inputFolder).ToList();
}
else
{
dirFileMessage = String.Format("Input directory {0} not found", workflow.inputFolder);
Log.Information(dirFileMessage);
}
// Step 4: If directory or files do not exist reschedule task
if (files.Count == 0)
{
dirFileMessage = (dirFileMessage.Length > 0) ? dirFileMessage : String.Format("No files found in directory: {0}", workflow.inputFolder);
await this.UpdateStatus(task.id, "SCHEDULED", dirFileMessage);
var newSchedule = new Hangfire.States.ScheduledState(TimeSpan.FromMinutes(workflow.interval));
task.start = DateTime.Now.AddMinutes(workflow.interval);
await _context.SaveChangesAsync();
try
{
BackgroundJobClient backgroundClient = new BackgroundJobClient();
backgroundClient.ChangeState(task.taskID, newSchedule);
Log.Information("Task Rescheduled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input Directory: {3}, Message: {4}", task.workflowID, task.id, task.taskID, workflow.inputFolder, "No files found in input directory.");
}
catch (Exception)
{
Log.Warning("Error attempting to reschedule Hangfire job. Workflow ID: {0}, task ID: {1}", task.workflowID, task.id);
}
return;
}
// Step 5: If file does exist, update task inputFile then compare against previous Tasks.
task.inputFile = files.First();
task.status = "PROCESSING";
await _context.SaveChangesAsync();
bool alreadyCompleted = await this.InputFileCheck(task.inputFile, task.workflowID);
if (alreadyCompleted)
{
Log.Information("Hash input file match for WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input File: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.inputFile, "Rerunning task after removing file.");
try
{
File.Delete(task.inputFile);
Log.Information("Hash input file match successfully deleted. WorkflowID: {0}, ID: {1}, Input File: {2}", task.workflowID, task.id, task.inputFile);
}
catch (FileNotFoundException ex)
{
Log.Warning("Error unable to delete input file after hash file match with previous input file. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
string statusMessage = String.Format("Input file: {0} matches previously processed input file", task.inputFile);
await this.UpdateStatus(task.id, "SCHEDULED", statusMessage);
await this.RunTask(task.id);
return;
}
ProcessorManager pm = new ProcessorManager();
string config = "./app_files/processors";
ProcessorDTO processor = pm.GetProcessors(config).Find(p => p.Name.ToLower() == workflow.processor.ToLower());
if (processor == null)
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}, Processor: {4}", task.workflowID, task.id, task.taskID, "Unable to find Processor for the Task.", workflow.processor);
await this.UpdateStatus(task.id, "CANCELLED", "Error, invalid processor name. Name: " + workflow.processor);
return;
}
try
{
// Step 6: Run processor on source file
if (!new DirectoryInfo(@workflow.outputFolder).Exists)
{
Directory.CreateDirectory(@workflow.outputFolder);
}
}
catch (UnauthorizedAccessException ex)
{
Log.Warning("Task unable to create output directory, unauthorized access exception. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Output Directory: {3}", task.workflowID, task.id, task.taskID, workflow.outputFolder);
}
Dictionary <string, ResponseMessage> outputs = new Dictionary <string, ResponseMessage>();
string file = task.inputFile;
DataTableResponseMessage result = pm.ExecuteProcessor(processor.Path, processor.Name, file);
GC.Collect();
GC.WaitForPendingFinalizers();
if (result.ErrorMessage == null && result.TemplateData != null)
{
var output = pm.WriteTemplateOutputFile(workflow.outputFolder, result.TemplateData);
outputs.Add(file, output);
}
else
{
string errorMessage = "";
string logMessage = "";
if (result.TemplateData == null)
{
errorMessage = "Processor results template data is null. ";
}
if (result.ErrorMessage != null)
{
errorMessage = errorMessage + result.ErrorMessage;
logMessage = errorMessage;
}
if (result.LogMessage != null)
{
logMessage = result.LogMessage;
}
await this.UpdateStatus(task.id, "CANCELLED", "Error processing data: " + errorMessage);
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, logMessage);
return;
}
// Step 7: Check if output file exists
bool processed = false;
for (int i = 0; i < outputs.Count; i++)
{
var output = outputs.ElementAt(i);
string outputPath = output.Value.OutputFile;
// Step 8: If output file does exist, update task outputFile and delete input file
if (File.Exists(outputPath))
{
processed = true;
string fileName = System.IO.Path.GetFileName(output.Key);
string inputPath = System.IO.Path.Combine(workflow.inputFolder, fileName);
DataBackup dbBackup = new DataBackup();
dbBackup.DumpData(id, inputPath, outputPath);
try
{
File.Delete(inputPath);
}
catch (Exception ex)
{
Log.Warning("Error unable to delete input file after successful processing. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
task.outputFile = outputPath;
await _context.SaveChangesAsync();
}
else
{
await this.UpdateStatus(task.id, "SCHEDULED", "Error unable to export output. Error Messages: " + output.Value.ErrorMessage);
}
}
// Step 9: Change task status to COMPLETED
// Step 10: Create new Task and schedule
string newStatus = "";
if (processed)
{
newStatus = "COMPLETED";
Log.Information("Task Completed. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID);
try
{
if (files.Count > 1)
{
await this.CreateNewTask(workflow.id, 0);
}
else
{
await this.CreateNewTask(workflow.id, workflow.interval);
}
}
catch (Exception)
{
Log.Warning("Error creating new Hangfire job after successful job. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
}
else
{
newStatus = "CANCELLED";
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Failed to process input file.");
}
await this.UpdateStatus(task.id, newStatus);
}
