/// <summary>
/// extract block from content
/// </summary>
/// <param name="content">content need to be extract</param>
/// <param name="block">extract block</param>
/// <returns></returns>
public static ExtractResult Extract(string content, ExtractBlock block)
{
var pr = ProcessorManager.Process(content, block.Selectors);
var result = new ExtractResult
{
Name = block.Name,
Content = pr.Content
};
if (block.Blocks != null && block.Blocks.Count > 0)
{
result.Blocks = Extract(result.Content.ToString(), block.Blocks);
}
if (block.TileSelector != null && block.TileSelector.Selectors.Count > 0)
{
result.Tiles = ExtractTile(pr.Content, block.TileSelector);
}
if (block.Metas.Count > 0)
{
result.Metas = ExtractMeta(pr.Content, block.Metas);
}
return(result);
}
private static void Execute([NotNull] PipelineStartInfo info)
{
using (new ProfileSection("Execute pipeline processors"))
{
ProfileSection.Argument("info", info);
try
{
if (info.PipelineController != null)
{
info.PipelineController.Maximum = ProcessorManager.GetProcessorsCount(info.ProcessorArgs, info._Steps);
}
bool result = ExecuteSteps(info.ProcessorArgs, info._Steps, info.PipelineController);
if (info.PipelineController != null)
{
info.PipelineController.Finish("Done.", result);
}
if (result)
{
info.ProcessorArgs.FireOnCompleted();
info.ProcessorArgs.Dispose();
}
}
catch (Exception ex)
{
Log.Warn(ex, "An error occurred during executing a pipeline");
info.ProcessorArgs.Dispose();
}
}
}
/// <summary>
/// extract base from content
/// </summary>
/// <param name="content">content need to be extract</param>
/// <param name="extractBase">extract base</param>
/// <returns>extract result collection</returns>
public static ExtractResultCollection ExtractSelector(string content, ExtractBase extractBase)
{
var pr = ProcessorManager.Process(content, extractBase.Selectors);
var results = new ExtractResultCollection();
foreach (var m in pr.Matches)
{
ExtractResult result;
try
{
result = new ExtractResult
{
Name = "tile",
Content = Convert.ChangeType(m, extractBase.DataType)
};
}
catch
{
result = new ExtractResult
{
Name = "tile",
Content = m
};
}
results.Add(result);
}
return(results);
}
/// <summary>
/// extract block from content
/// </summary>
/// <param name="content">content need to be extract</param>
/// <param name="block">extract block</param>
/// <returns></returns>
public static ExtractResult Extract(string content, ExtractBlock block)
{
var pr = ProcessorManager.Process(content, block.Selectors);
var result = new ExtractResult
{
Name = block.Name,
Content = pr.Content
};
if (block.Blocks != null && block.Blocks.Count > 0)
{
result.Blocks = Extract(result.Content.ToString(), block.Blocks);
var paging = result.Blocks.SingleOrDefault(m => m.Name == "_paging");
if (paging != null && paging.Tiles.Count == 0)
{
paging = Extract(content, block.Blocks.Single(m => m.Name == "_paging"));
}
}
if (block.TileSelector != null && block.TileSelector.Selectors.Count > 0)
{
result.Tiles = ExtractTile(pr.Content, block.TileSelector);
}
if (block.Metas.Count > 0)
{
result.Metas = ExtractMeta(pr.Content, block.Metas);
}
return(result);
}
public void DeployPatternToProcessorGroup(ProcessorManager[,] processorGroup, IProcessorFSM processorGroupFSM)
{
this.processorGroup = processorGroup;
this.processorGroupFSM = processorGroupFSM;
processorGroupFSM.SetStateTimes(patternDifficulty.GetProcessorFSMTimers());
resetConfigurator();
int pattern = patternDifficulty.GetNewProcessorGroupPattern();
ConfigureProcessorsBasedOnPattern(SelectPatternFunction(pattern));
}
private void btnRun_Click(object sender, EventArgs e)
{
DataTableResponseMessage dtRespMsg = null;
try
{
templateDataGridView.DataSource = null;
ClearMessage();
UserMessage("Running");
ProcessorDTO proc = comboBox1.Items[comboBox1.SelectedIndex] as ProcessorDTO;
txtID.Text = proc.UniqueId;
txtName.Text = proc.Name;
txtDesc.Text = proc.Description;
txtFileType.Text = proc.InstrumentFileType;
ProcessorManager procMgr = new ProcessorManager();
//string output = @"E:\lims\LIMSDesktop\bin\Debug\netcoreapp3.0\Processors\Output\file.csv";
//string procPaths = @"E:\lims\lims_server\app_files\processors";
dtRespMsg = procMgr.ExecuteProcessor(proc.Path, txtName.Text, txtInput.Text);
GC.Collect();
GC.WaitForPendingFinalizers();
if (dtRespMsg == null)
{
//dtRespMsg = new DataTableResponseMessage();
UserMessage(string.Format("Error processing file {0} with processor {1}", txtInput.Text, txtName.Text));
//dtRespMsg.ErrorMessage = string.Format("Error processing file {0} with processor {1}", txtInput.Text, txtName.Text);
}
if (dtRespMsg.ErrorMessage != null)
{
LogMessage(dtRespMsg.ErrorMessage);
}
if (!string.IsNullOrWhiteSpace(dtRespMsg.LogMessage))
{
LogMessage(dtRespMsg.LogMessage);
}
if (dtRespMsg.TemplateData != null)
{
templateDataGridView.DataSource = dtRespMsg.TemplateData;
UserMessage("Success");
}
}
catch (Exception ex)
{
LogMessage(string.Format("Error executing processor {0} with input {1}", txtID.Text, txtInput.Text));
LogMessage($"Error: {ex.Message}");
if (dtRespMsg != null && dtRespMsg.LogMessage != null)
{
LogMessage(dtRespMsg.LogMessage);
}
}
}
private void Form1_Load(object sender, EventArgs e)
{
ProcessorManager procMgr = new ProcessorManager();
string basePath = Assembly.GetExecutingAssembly().Location;
string baseFolder = Path.GetDirectoryName(basePath);
string procPaths = Path.Combine(baseFolder, "app_files\\processors");
var lstProc = procMgr.GetProcessors(procPaths);
comboBox1.DataSource = lstProc;
comboBox1.DisplayMember = "Name";
//comboBox1.ValueMember = "Processor";
}
/// <summary>
/// Constructor. Initialize the processor manager and register processors.
/// </summary>
public RpcCommandHandler(CommandFactory commandFactory)
{
this.commandFactory = commandFactory;
processorManager = new ProcessorManager();
//process request
processorManager.registerProcessor(RpcCommandCode.RPC_REQUEST, new RpcRequestProcessor(this.commandFactory));
//process response
processorManager.registerProcessor(RpcCommandCode.RPC_RESPONSE, new RpcResponseProcessor());
processorManager.registerProcessor(CommonCommandCode.HEARTBEAT, new RpcHeartBeatProcessor());
processorManager.registerDefaultProcessor(new AbstractRemotingProcessorAnonymousInnerClass(this));
}
private static IEnumerable <Step> CreateStepsPrivate([NotNull] IEnumerable <StepDefinition> steps, [NotNull] ProcessorArgs args, [CanBeNull] IPipelineController controller = null)
{
Assert.ArgumentNotNull(steps, nameof(steps));
Assert.ArgumentNotNull(args, nameof(args));
foreach (StepDefinition stepDefinition in steps)
{
var argsName = stepDefinition.ArgsName.EmptyToNull();
Step step = new Step(ProcessorManager.CreateProcessors(stepDefinition.ProcessorDefinitions, args, controller), argsName);
Assert.IsNotNull(step, "Can't instantiate step");
yield return(step);
}
}
public virtual void testRegisterProcessor()
{
ProcessorManager processorManager = new ProcessorManager();
CommandCode cmd1 = RpcCommandCode.RPC_REQUEST;
CommandCode cmd2 = RpcCommandCode.RPC_REQUEST;
RpcRequestProcessor rpcRequestProcessor1 = new RpcRequestProcessor();
RpcRequestProcessor rpcRequestProcessor2 = new RpcRequestProcessor();
processorManager.registerProcessor(cmd1, rpcRequestProcessor1);
processorManager.registerProcessor(cmd2, rpcRequestProcessor2);
Assert.Equal(processorManager.getProcessor(cmd1), rpcRequestProcessor2);
Assert.Equal(processorManager.getProcessor(cmd2), rpcRequestProcessor2);
}
private void Awake()
{
if (Instance == null)
{
DontDestroyOnLoad(gameObject);
processorManager = new ProcessorManager(PlayerPrefs.GetString(AcessPython.KEYPATHPYTHON));
run = false;
contVertx = 0;
}
else
{
Destroy(this);
}
}
public void SetCycleCompletion(ProcessorManager processor, float cyclePercent)
{
transitions[ProcessorState.CoolingDown](processor);
ProcessorState targetState = ProcessorState.Cool;
float targetStateStayTime = stateTimers[targetState];
while (targetStateStayTime < totalCycleTime * cyclePercent)
{
transitions[targetState](processor);
++targetState;
targetStateStayTime += stateTimers[targetState];
}
processor.state = targetState;
processor.stateExitTime = Time.timeSinceLevelLoad + (targetStateStayTime - (totalCycleTime * cyclePercent));
}
private void btnSave_Click(object sender, EventArgs e)
{
//ClearMessage();
string dtName = "";
try
{
//SaveFileDialog sfd = new SaveFileDialog();
FolderBrowserDialog fbd = new FolderBrowserDialog();
fbd.Description = "Save output file location";
fbd.ShowNewFolderButton = true;
string fName = Path.GetFileNameWithoutExtension(txtInput.Text);
//sfd.FileName = fName;
//sfd.Filter = @"Excel Files (*.xlsx)|*.xlsx";
DialogResult dr = fbd.ShowDialog();
if (dr != DialogResult.OK)
{
return;
}
ProcessorManager procMgr = new ProcessorManager();
DataTable dt = templateDataGridView.DataSource as DataTable;
if (dt != null)
{
dtName = dt.TableName;
}
//string dir = Path.GetDirectoryName(sfd.FileName);
//string outPath = Path.Combine(sfd.FileName, "output");
DirectoryInfo di = new DirectoryInfo(fbd.SelectedPath);
if (!di.Exists)
{
di.Create();
}
procMgr.WriteTemplateOutputFile(fbd.SelectedPath, dt);
UserMessage("Success");
}
catch (Exception ex)
{
LogMessage(string.Format("Error saving output {0}", dtName));
LogMessage(ex.Message);
}
}
public NetworkServer(ILogger logger, PacketFactory packetFactory, ProcessorManager processorManager)
{
this.logger = logger;
bossGroup = new MultithreadEventLoopGroup(1);
workerGroup = new MultithreadEventLoopGroup();
bootstrap = new ServerBootstrap()
.Option(ChannelOption.SoBacklog, 100)
.Group(bossGroup, workerGroup)
.Channel <TcpServerSocketChannel>()
.ChildHandler(new ActionChannelInitializer <IChannel>(x =>
{
var pipeline = x.Pipeline;
var client = new NetworkClient(logger, x);
var session = new WorldSession(client);
client.PacketReceived += packet =>
{
var processor = processorManager.GetPacketProcessor(packet.GetType());
if (processor is null)
{
return(Task.CompletedTask);
}
return(processor.ProcessPacket(session, packet));
};
client.Disconnected += () =>
{
return(Task.CompletedTask);
};
pipeline.AddLast("decoder", new Decoder(logger, client));
pipeline.AddLast("deserializer", new Deserializer(logger, packetFactory));
pipeline.AddLast("client", client);
pipeline.AddLast("encoder", new Encoder(logger));
pipeline.AddLast("serializer", new Serializer(logger, packetFactory));
}));
}
public void UpdateHeatupPhase(ProcessorManager processor)
{
switch(processor.state)
{
case ProcessorState.Cool:
StayCool(processor);
break;
case ProcessorState.HeatingUp:
HeatUp(processor);
break;
case ProcessorState.Hot:
StayHot(processor);
break;
case ProcessorState.CoolingDown:
CoolDown(processor);
break;
}
}
private void Awake()
{
_myID = getNetworkID();
isConnect = _myID >= 0;
if (!isConnect)
{
instance = null;
}
else
{
instance = this;
networkStored = NetworkStoredData.instance;
myTeam = NetworkStoredData.instance ? NetworkStoredData.instance.GetTeam(_myID) : 0;
_localPlayerMovement = _localPlayer.GetComponent <PlayerMovement>();
_localPlayer.GetComponent <PlayerScoreTracker>().Init(_myID, true);
_players.Insert(_myID, _localPlayer);
_processor = gameObject.transform.GetChild(0).gameObject.GetComponent <ProcessorManager>();
_processor.StartProcessing();
SpawnPlayerOnNetwork();
}
}
public ImageProcessorViewModel()
{
manager = new ProcessorManager(this);
LoadImageCommand = new RelayCommand(LoadNewImage);
SaveImageCommand = new RelayCommand(SaveImage);
BinarizeCommand = new RelayCommand(TrigerImageOperation(Operations[1]));
ToGrayWithEyeAdaptationCommand = new RelayCommand(TrigerImageOperation(Operations[2]));
ToGrayAverageCommand = new RelayCommand(TrigerImageOperation(Operations[3]));
EdgeDetectLaplaceCommand = new RelayCommand(TrigerImageOperation(Operations[4]));
EdgeDetectDiagonalCommand = new RelayCommand(TrigerImageOperation(Operations[5]));
EdgeDetectHorizontalCommand = new RelayCommand(TrigerImageOperation(Operations[6]));
EdgeDetectVerticalCommand = new RelayCommand(TrigerImageOperation(Operations[7]));
SculptureEastCommand = new RelayCommand(TrigerImageOperation(Operations[8]));
SculptureSouthEastCommand = new RelayCommand(TrigerImageOperation(Operations[9]));
UpperProofFilterCommand = new RelayCommand(TrigerImageOperation(Operations[10]));
LowerProofFilterBlurNCommand = new RelayCommand(TrigerImageOperation(Operations[11]));
LowerProofFilterGaussCommand = new RelayCommand(TrigerImageOperation(Operations[12]));
SetOperationToDefaultFilterCommand = new RelayCommand((o) => CurrentOperation = Operations[13]);
DefaultFilterGaussCommand = new RelayCommand(TrigerImageOperation(Operations[13]));
HistogramStretchingCommand = new RelayCommand(TrigerImageOperation(Operations[14]));
ShowOrginalImageCommand = new RelayCommand((o => manager.Reset()));
NegationCommand = new RelayCommand(TrigerImageOperation(Operations[15]));
CalculateHistogramCommand = new RelayCommand(CalculateHistogram);
HistogramEqualizationCommand = new RelayCommand(TrigerImageOperation(Operations[16]));
BrightnessOperation = new BrightnessImageOperation();
BrightnessCommand = new RelayCommand(TrigerImageOperation(BrightnessOperation));
ContrastOperation = new ContrastImageOperation();
ContrastCommand = new RelayCommand(TrigerImageOperation(ContrastOperation));
ThresholdingOperation = new ThresholdingImageOperation();
ThresholdingCommand = new RelayCommand(TrigerImageOperation(ThresholdingOperation));
}
/// <summary>
/// extract tile from content
/// </summary>
/// <param name="content">content need to be extract</param>
/// <param name="tile">extract tile</param>
/// <returns>extract result collection</returns>
public static ExtractResultCollection ExtractTile(string content, ExtractTile tile)
{
var pr = ProcessorManager.Process(content, tile.Selectors);
var results = new ExtractResultCollection();
foreach (var m in pr.Matches)
{
var result = new ExtractResult
{
Name = "tile",
Content = m
};
if (tile.Metas.Count > 0)
{
result.Metas = ExtractMeta(m, tile.Metas);
}
results.Add(result);
}
return(results);
}
private void TransitionToCool(ProcessorManager processor)
{
SetStateAndTimer(processor, ProcessorState.Cool);
}
public async System.Threading.Tasks.Task RunTask(string id)
{
var task = await _context.Tasks.SingleAsync(t => t.id == id);
Log.Information("Executing Task. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID);
// Step 1: If status!="SCHEDULED" cancel task
if (!task.status.Equals("SCHEDULED"))
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Current Status: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.status, "Task status is not marked as schedulled.");
await this.UpdateStatus(task.id, "CANCELLED", "Task status was set to: " + task.status);
return;
}
// Step 2: Change status to "STARTING"
await this.UpdateStatus(task.id, "STARTING", "");
var workflow = await _context.Workflows.Where(w => w.id == task.workflowID).FirstOrDefaultAsync();
if (workflow == null)
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Unable to find Workflow for the Task.");
await this.UpdateStatus(task.id, "CANCELLED", "Error attempting to get workflow of this task. Workflow ID: " + task.workflowID);
return;
}
// Step 3: Check source directory for files
List <string> files = new List <string>();
string dirFileMessage = "";
if (new DirectoryInfo(@workflow.inputFolder).Exists)
{
files = Directory.GetFiles(@workflow.inputFolder).ToList();
}
else
{
dirFileMessage = String.Format("Input directory {0} not found", workflow.inputFolder);
Log.Information(dirFileMessage);
}
// Step 4: If directory or files do not exist reschedule task
if (files.Count == 0)
{
dirFileMessage = (dirFileMessage.Length > 0) ? dirFileMessage : String.Format("No files found in directory: {0}", workflow.inputFolder);
await this.UpdateStatus(task.id, "SCHEDULED", dirFileMessage);
var newSchedule = new Hangfire.States.ScheduledState(TimeSpan.FromMinutes(workflow.interval));
task.start = DateTime.Now.AddMinutes(workflow.interval);
await _context.SaveChangesAsync();
try
{
BackgroundJobClient backgroundClient = new BackgroundJobClient();
backgroundClient.ChangeState(task.taskID, newSchedule);
Log.Information("Task Rescheduled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input Directory: {3}, Message: {4}", task.workflowID, task.id, task.taskID, workflow.inputFolder, "No files found in input directory.");
}
catch (Exception)
{
Log.Warning("Error attempting to reschedule Hangfire job. Workflow ID: {0}, task ID: {1}", task.workflowID, task.id);
}
return;
}
// Step 5: If file does exist, update task inputFile then compare against previous Tasks.
task.inputFile = files.First();
task.status = "PROCESSING";
await _context.SaveChangesAsync();
bool alreadyCompleted = await this.InputFileCheck(task.inputFile, task.workflowID);
if (alreadyCompleted)
{
Log.Information("Hash input file match for WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input File: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.inputFile, "Rerunning task after removing file.");
try
{
File.Delete(task.inputFile);
Log.Information("Hash input file match successfully deleted. WorkflowID: {0}, ID: {1}, Input File: {2}", task.workflowID, task.id, task.inputFile);
}
catch (FileNotFoundException ex)
{
Log.Warning("Error unable to delete input file after hash file match with previous input file. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
string statusMessage = String.Format("Input file: {0} matches previously processed input file", task.inputFile);
await this.UpdateStatus(task.id, "SCHEDULED", statusMessage);
await this.RunTask(task.id);
return;
}
ProcessorManager pm = new ProcessorManager();
string config = "./app_files/processors";
ProcessorDTO processor = pm.GetProcessors(config).Find(p => p.Name.ToLower() == workflow.processor.ToLower());
if (processor == null)
{
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}, Processor: {4}", task.workflowID, task.id, task.taskID, "Unable to find Processor for the Task.", workflow.processor);
await this.UpdateStatus(task.id, "CANCELLED", "Error, invalid processor name. Name: " + workflow.processor);
return;
}
try
{
// Step 6: Run processor on source file
if (!new DirectoryInfo(@workflow.outputFolder).Exists)
{
Directory.CreateDirectory(@workflow.outputFolder);
}
}
catch (UnauthorizedAccessException ex)
{
Log.Warning("Task unable to create output directory, unauthorized access exception. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Output Directory: {3}", task.workflowID, task.id, task.taskID, workflow.outputFolder);
}
Dictionary <string, ResponseMessage> outputs = new Dictionary <string, ResponseMessage>();
string file = task.inputFile;
DataTableResponseMessage result = pm.ExecuteProcessor(processor.Path, processor.Name, file);
GC.Collect();
GC.WaitForPendingFinalizers();
if (result.ErrorMessage == null && result.TemplateData != null)
{
var output = pm.WriteTemplateOutputFile(workflow.outputFolder, result.TemplateData);
outputs.Add(file, output);
}
else
{
string errorMessage = "";
string logMessage = "";
if (result.TemplateData == null)
{
errorMessage = "Processor results template data is null. ";
}
if (result.ErrorMessage != null)
{
errorMessage = errorMessage + result.ErrorMessage;
logMessage = errorMessage;
}
if (result.LogMessage != null)
{
logMessage = result.LogMessage;
}
await this.UpdateStatus(task.id, "CANCELLED", "Error processing data: " + errorMessage);
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, logMessage);
return;
}
// Step 7: Check if output file exists
bool processed = false;
for (int i = 0; i < outputs.Count; i++)
{
var output = outputs.ElementAt(i);
string outputPath = output.Value.OutputFile;
// Step 8: If output file does exist, update task outputFile and delete input file
if (File.Exists(outputPath))
{
processed = true;
string fileName = System.IO.Path.GetFileName(output.Key);
string inputPath = System.IO.Path.Combine(workflow.inputFolder, fileName);
DataBackup dbBackup = new DataBackup();
dbBackup.DumpData(id, inputPath, outputPath);
try
{
File.Delete(inputPath);
}
catch (Exception ex)
{
Log.Warning("Error unable to delete input file after successful processing. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
task.outputFile = outputPath;
await _context.SaveChangesAsync();
}
else
{
await this.UpdateStatus(task.id, "SCHEDULED", "Error unable to export output. Error Messages: " + output.Value.ErrorMessage);
}
}
// Step 9: Change task status to COMPLETED
// Step 10: Create new Task and schedule
string newStatus = "";
if (processed)
{
newStatus = "COMPLETED";
Log.Information("Task Completed. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID);
try
{
if (files.Count > 1)
{
await this.CreateNewTask(workflow.id, 0);
}
else
{
await this.CreateNewTask(workflow.id, workflow.interval);
}
}
catch (Exception)
{
Log.Warning("Error creating new Hangfire job after successful job. Workflow ID: {0}, ID: {1}", task.workflowID, task.id);
}
}
else
{
newStatus = "CANCELLED";
Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Failed to process input file.");
}
await this.UpdateStatus(task.id, newStatus);
}
private void CoolDown(ProcessorManager processor)
{
float coolDownPercent = TimerProgressPercent(processor, stateTimers[ProcessorState.CoolingDown]);
processor.TintProcessorSprite(maxHeatupColor, Color.white, coolDownPercent);
if(coolDownPercent >= 0.99f)
{
processor.SetProcessorSpriteColor(Color.white);
TransitionToCool(processor);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="HubConfigurator"/> class.
/// </summary>
/// <param name="processorManager">
/// The processor manager.
/// </param>
public HubConfigurator(ProcessorManager processorManager)
{
this.processorManager = processorManager;
}
private void HeatUp(ProcessorManager processor)
{
float heatUpPercent = TimerProgressPercent(processor, stateTimers[ProcessorState.HeatingUp]);
processor.TintProcessorSprite(Color.white, maxHeatupColor, heatUpPercent);
if(heatUpPercent >= 0.99f)
{
processor.SetProcessorSpriteColor(Color.red);
TransitionToHot(processor);
}
}
private void TransitionToHot(ProcessorManager processor)
{
processor.SetHazadrousLayer();
SetStateAndTimer(processor, ProcessorState.Hot);
}
private void SettingsForm_Load(object sender, EventArgs e)
{
Lpp.Dns.DataMart.Client.DomainManger.DomainManager domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath);
try
{
if (ModelDescription == null)
{
return;
}
if (ModelDescription.ProcessorId == Guid.Empty)
{
ModelDescription.ProcessorId = HubModel.ModelProcessorId;
}
//get the package identifier and version
var packageIdentifier = DnsServiceManager.GetRequestTypeIdentifier(NetworkSetting, ModelDescription.ModelId, ModelDescription.ProcessorId);
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(NetworkSetting, packageIdentifier);
}
domainManager.Load(packageIdentifier.Identifier, packageIdentifier.Version);
modelProcessor = domainManager.GetProcessor(ModelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(ModelDescription, modelProcessor);
if (modelProcessor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)modelProcessor).Initialize(ModelDescription.ModelId, Array.Empty <DocumentWithStream>());
}
this.SuspendLayout();
if (modelProcessor.ModelMetadata.Settings == null || !modelProcessor.ModelMetadata.Settings.Any())
{
var noSettingsLabel = new Label();
noSettingsLabel.Text = "This model processor does not have any settings.";
noSettingsLabel.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
noSettingsLabel.TextAlign = ContentAlignment.MiddleCenter;
tableLayoutPanel1.Controls.Add(noSettingsLabel, 0, 0);
tableLayoutPanel1.SetColumnSpan(noSettingsLabel, 2);
}
else
{
_editors = new HashSet <Control>();
tableLayoutPanel1.RowStyles.Clear();
int rowIndex = 0;
IEnumerable <Lpp.Dns.DataMart.Model.Settings.ProcessorSetting> settings = modelProcessor.ModelMetadata.Settings;
if (modelProcessor.ModelMetadata.SQlProviders != null && modelProcessor.ModelMetadata.SQlProviders.Any())
{
DataMart.Client.Controls.DataSourceEditor dsEditor = new Controls.DataSourceEditor(modelProcessor.ModelMetadata, ModelDescription.Properties);
dsEditor.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize));
tableLayoutPanel1.Controls.Add(dsEditor, 0, rowIndex++);
tableLayoutPanel1.SetColumnSpan(dsEditor, 2);
settings = settings.Where(s => !Lpp.Dns.DataMart.Model.Settings.ProcessorSettings.IsDbSetting(s.Key)).ToArray();
_editors.Add(dsEditor);
}
settings.ToList().ForEach(s => {
string value = ModelDescription.Properties.Where(p => string.Equals(p.Name, s.Key, StringComparison.OrdinalIgnoreCase)).Select(p => p.Value).FirstOrDefault();
if (string.IsNullOrEmpty(value) && s.Required && !string.IsNullOrEmpty(s.DefaultValue))
{
value = s.DefaultValue;
}
Label lbl = new Label();
lbl.AutoSize = true;
lbl.Anchor = AnchorStyles.Right;
lbl.TextAlign = ContentAlignment.MiddleRight;
lbl.Text = s.Title.EndsWith(":") ? s.Title : s.Title + ":";
Control editor = null;
if (s.ValidValues != null)
{
ComboBox combo = new ComboBox();
combo.DropDownStyle = ComboBoxStyle.DropDownList;
combo.Anchor = AnchorStyles.Right | AnchorStyles.Left;
combo.Name = s.Key;
combo.Items.AddRange(s.ValidValues.Select(v => new PropertyData(v.Key, v.Value.ToString())).ToArray());
if (!string.IsNullOrEmpty(value))
{
foreach (PropertyData p in combo.Items)
{
if (string.Equals(p.Value, value, StringComparison.OrdinalIgnoreCase))
{
combo.SelectedItem = p;
break;
}
}
}
if (combo.SelectedIndex < 0)
{
combo.SelectedIndex = 0;
}
editor = combo;
}
else
{
if (s.ValueType == typeof(bool) || s.ValueType == typeof(Nullable <bool>))
{
CheckBox chkbox = new CheckBox();
chkbox.Anchor = AnchorStyles.Left;
chkbox.Text = s.Title;
chkbox.TextAlign = ContentAlignment.MiddleLeft;
chkbox.AutoSize = true;
if (!string.IsNullOrEmpty(value))
{
bool isChecked = false;
bool.TryParse(value, out isChecked);
chkbox.Checked = isChecked;
}
editor = chkbox;
lbl = null;
}
else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FilePickerEditor))
{
SelectFileButton btn = new SelectFileButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FilePickerEditor);
if (btn.Multiselect)
{
btn.FileNames = ((value ?? "").Trim(',')).Split(',');
}
else
{
btn.FileName = value;
}
btn.Anchor = AnchorStyles.Right | AnchorStyles.Left;
editor = btn;
}
else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor))
{
SelectFolderButton btn = new SelectFolderButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor);
btn.FolderPath = value;
btn.Anchor = AnchorStyles.Right | AnchorStyles.Left;
editor = btn;
}
else
{
TextBox txtbox = new TextBox();
txtbox.Anchor = AnchorStyles.Right | AnchorStyles.Left;
txtbox.Text = value;
editor = txtbox;
}
}
if (editor != null)
{
editor.Tag = s.Key;
_editors.Add(editor);
tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize));
if (lbl != null)
{
tableLayoutPanel1.Controls.Add(lbl, 0, rowIndex);
tableLayoutPanel1.Controls.Add(editor, 1, rowIndex++);
}
else
{
tableLayoutPanel1.Controls.Add(editor, 0, rowIndex++);
tableLayoutPanel1.SetColumnSpan(editor, 2);
}
}
});
//add auto expanding row to bottom to fill empty space
Label emptyLabel = new Label();
emptyLabel.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
emptyLabel.Text = string.Empty;
tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.Percent, 100f));
tableLayoutPanel1.Controls.Add(emptyLabel, 0, rowIndex);
tableLayoutPanel1.SetColumnSpan(emptyLabel, 2);
}
this.ResumeLayout(true);
}
catch (Exception ex)
{
log.Error(ex);
Close();
}
finally
{
domainManager.Dispose();
this.Cursor = Cursors.Default;
}
}
protected override async System.Threading.Tasks.Task ExecuteAsync(CancellationToken stoppingToken)
{
string projectRootPath = _hostingEnvironment.ContentRootPath;
string processorPath = Path.Combine(projectRootPath, "app_files", "processors");
PluginBase.ProcessorManager pm = new ProcessorManager();
var procsFromDisk = pm.GetProcessors(processorPath);
List <string> procsDiskNames = new List <string>();
foreach (ProcessorDTO proc in procsFromDisk)
{
procsDiskNames.Add(proc.Name.ToLower());
}
// Create a new scope to retrieve scoped services
List <string> procsInDb = new List <string>();
using (var scope = _serviceProvider.CreateScope())
{
// Get the DbContext instance
var procService = scope.ServiceProvider.GetRequiredService <IProcessorService>();
//var dbProcs = await procService.GetAll();
var dbProcs = await procService.GetAll();
//var dbProcs = procs.Result;
List <Processor> lstProcs = new List <Processor>();
foreach (Processor proc in dbProcs)
{
proc.enabled = false;
lstProcs.Add(proc);
procsInDb.Add(proc.name.ToLower());
}
//Set all the processors in the db to enabled = false
await procService.Update(lstProcs.ToArray());
var procsIntersect = procsDiskNames.Intersect(procsInDb);
lstProcs = new List <Processor>();
foreach (string proc in procsIntersect)
{
var processor = await procService.GetById(proc);
//Processor processor = result.Result;
processor.enabled = true;
lstProcs.Add(processor);
}
//Set all the processors in the db that are also on disk to enabled = true
await procService.Update(lstProcs.ToArray());
var newProcs = procsDiskNames.Except(procsInDb);
List <ProcessorDTO> lstProcDTO = new List <ProcessorDTO>();
foreach (string proc in newProcs)
{
lstProcDTO.Add(procsFromDisk.Find(x => x.Name.ToLower() == proc.ToLower()));
}
foreach (ProcessorDTO proc in lstProcDTO)
{
Processor processor = new Processor();
processor.enabled = true;
processor.description = proc.Description;
processor.file_type = proc.InstrumentFileType;
processor.id = proc.UniqueId.ToLower();
processor.name = proc.Name.ToLower();
await procService.Create(processor);
}
}
//return System.Threading.Tasks.Task.CompletedTask;
return;
}
public Universe()
{
Entities = new EntityManager();
Components = new ComponentManager(Entities);
Processors = new ProcessorManager(this);
}
private void AutoProcess(KeyValuePair <string, HubRequest> input)
{
var request = input.Value;
var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
if (datamartDescription.ProcessQueriesAndUploadAutomatically == false && datamartDescription.ProcessQueriesAndNotUpload == false)
{
//just notify, do not process
string message = $"New query submitted and awaiting processing in { request.ProjectName } Project: { request.Source.Name } ({request.Source.Identifier})";
SystemTray.DisplayNewQueryNotificationToolTip(message);
RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);
return;
}
var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);
var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
};
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(_networkSetting, packageIdentifier);
}
var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath);
try
{
domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
processor.Settings.Add("NetworkId", request.NetworkId);
Lib.Caching.DocumentCacheManager cache = new Lib.Caching.DocumentCacheManager(request.NetworkId, request.DataMartId, request.Source.ID, request.Source.Responses.Where(x => x.DataMartID == request.DataMartId).FirstOrDefault().ResponseID);
//need to initialize before checking the capabilities and settings of the processor since they may change based on the type of request being sent.
if (processor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, _networkSetting))).ToArray());
}
if (processor != null &&
processor.ModelMetadata != null &&
processor.ModelMetadata.Capabilities != null &&
processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
!(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
{
//can't be run, don't attempt autoprocessing
RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);
domainManager.Dispose();
return;
}
request.Processor = processor;
if (cache.HasResponseDocuments == false && (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || request.RoutingStatus == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted))
{
if (processor != null)
{
SystemTray.GenerateNotification(request, request.NetworkId);
StartProcessingRequest(request, processor, datamartDescription, domainManager, cache);
return;
}
}
else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
{
if (datamartDescription.ProcessQueriesAndUploadAutomatically)
{
// Increment counter
System.Threading.Interlocked.Increment(ref _queriesProcessedCount);
SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId);
StartUploadingRequest(request, domainManager, cache);
return;
}
}
else if (cache.HasResponseDocuments)
{
RequestStatuses.TryAdd(input.Key, ProcessingStatus.PendingUpload);
}
domainManager.Dispose();
}
catch (Exception ex)
{
Log.Error($"Error autoprocessing Request: { request.Source.Identifier }, DataMartId: { request.DataMartId }, NetworkId: {request.NetworkId}", ex);
domainManager.Dispose();
throw;
}
}
public void Init()
{
testProcessorFSM = new ProcessorFSM ();
testProcessor = Resources.Load ("Processor") as GameObject;
testProcessorManager = testProcessor.GetComponent<ProcessorManager> ();
}
public void initialize(ProcessorManager[,] processorGroup)
{
this.processorGroup = processorGroup;
RrepartentProcessors();
ProcessorGroupInitialSetup();
}
private void TransitionToCoolingDown(ProcessorManager processor)
{
processor.SetSafeLayer();
SetStateAndTimer(processor, ProcessorState.CoolingDown);
}
private void SetStateAndTimer(ProcessorManager processor, ProcessorState state)
{
processor.stateExitTime = Time.timeSinceLevelLoad + stateTimers[state];
processor.state = state;
}
private void TransitionToHeatingUp(ProcessorManager processor)
{
SetStateAndTimer(processor, ProcessorState.HeatingUp);
}
protected override void OnDoWork(DoWorkEventArgs e)
{
_log.Info("Automated processing worker started.");
while (true)
{
_log.Info("Checking Master Request Queue for requests requiring automated processing.");
SystemTray.UpdateNotificationIcon(IconType.IconBusy, "Processing Requests");
var reqs = from ns in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Cast <NetWorkSetting>().ToObservable()
where ns.NetworkStatus == Util.ConnectionOKStatus
let dmIds = ns.DataMartList
// BMS: Note the ProcessAndNotUpload feature has been temporarilty disabled until we fix the processing around this feature
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || /* dm.ProcessQueriesAndNotUpload || */ dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId)
.ToArray()
where dmIds.Any()
from list in DnsServiceManager.GetRequestList(ns, 0, Properties.Settings.Default.AutoProcessingBatchSize,
new RequestFilter {
Statuses = new [] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted }, DataMartIds = dmIds
}, null, null)
from rl in list.Segment.EmptyIfNull().ToObservable()
where rl.AllowUnattendedProcessing
from r in RequestCache.ForNetwork(ns).LoadRequest(rl.ID, rl.DataMartID)
select new { Request = r, NetworkSetting = ns };
reqs
.Do(r =>
{
var request = r.Request;
var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);
var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
};
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(r.NetworkSetting, packageIdentifier);
}
using (var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath))
{
domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
if (processor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, r.NetworkSetting))).ToArray());
}
if (processor != null &&
processor.ModelMetadata != null &&
processor.ModelMetadata.Capabilities != null &&
processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
!(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
{
//can't be run, don't attempt autoprocessing
return;
}
request.Processor = processor;
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted)
{
if (processor != null)
{
SystemTray.generate_notification(request, request.NetworkId);
if (datamartDescription.NotifyOfNewQueries)
{
SystemTray.UpdateNotificationIcon(IconType.IconBusy, string.Format("Query Submitted by {0}", request.Source.Author.Username));
}
else
{
processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties);
ProcessRequest(request);
var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage))
{
// Post process requests that are automatically uploaded
processor.PostProcess(request.Source.ID.ToString());
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
}
else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
{
if (datamartDescription.ProcessQueriesAndUploadAutomatically)
{
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
})
.LogExceptions(_log.Error)
.Catch()
.LastOrDefault();
SystemTray.UpdateNotificationIcon(IconType.IconDefault, null);
Thread.Sleep(DMClient.Properties.Settings.Default.RefreshRate);
}
}
private void StayCool(ProcessorManager processor)
{
if(Time.timeSinceLevelLoad >= processor.stateExitTime)
{
TransitionToHeatingUp(processor);
}
}
private void StayHot(ProcessorManager processor)
{
if(Time.timeSinceLevelLoad >= processor.stateExitTime)
{
TransitionToCoolingDown(processor);
}
}
private float TimerProgressPercent(ProcessorManager processor, float stateStayTime)
{
float timeRemaining = processor.stateExitTime - Time.timeSinceLevelLoad;
float timeElapsed = stateStayTime - timeRemaining;
return timeElapsed / stateStayTime;
}
public static XmlElement Initialize(XmlElement pipelinesNode)
{
Log.Debug("Pipelines RAW configuration: {0}", pipelinesNode.OuterXml);
Definitions.Clear();
var resultXmlConfig = XmlDocumentEx.LoadXml("<pipelines />");
foreach (XmlElement element in pipelinesNode.ChildNodes.OfType <XmlElement>())
{
string pipelineName = element.Name;
string title = element.GetAttribute("title");
Assert.IsNotNullOrEmpty(title, "The '{0}' pipeline definition doesn't contain the title attribute".FormatWith(pipelineName));
var pipelineNode = resultXmlConfig.DocumentElement.AddElement(pipelineName);
pipelineNode.SetAttribute("title", title);
List <StepDefinition> steps = new List <StepDefinition>();
XmlNodeList stepNodes = element.SelectNodes("step");
// Processors are placed into steps
if (stepNodes != null && stepNodes.Count > 0)
{
foreach (XmlElement step in stepNodes.OfType <XmlElement>())
{
// Clever mechanism of steps reusing. Doesn't seem to be used somewhere.
string fromPipeline = step.GetAttribute("pipeline");
string args = step.GetAttribute("args").EmptyToNull();
if (!string.IsNullOrEmpty(fromPipeline))
{
PipelineDefinition def = Definitions[fromPipeline];
if (args != null)
{
def.Steps.ForEach(s => s.ArgsName = args);
}
AddSteps(steps, def);
}
// All step's processors are interted to the step
else
{
var stepNode = pipelineNode.AddElement("step");
stepNode.SetAttribute("args", args);
List <ProcessorDefinition> processorDefinitions = ProcessorManager.GetProcessors(step, pipelineName, stepNode);
AddStep(steps, processorDefinitions);
}
}
}
// Otherwise all the processors are placed into the default step.
else
{
List <ProcessorDefinition> processorDefinitions = ProcessorManager.GetProcessors(element, pipelineName, pipelineNode);
steps.Add(new StepDefinition(processorDefinitions));
}
Definitions.Add(pipelineName, new PipelineDefinition
{
Steps = steps,
Title = title
});
}
return(resultXmlConfig.DocumentElement);
}
// TODO Configure ProcessorConfigurator
/// <summary>
/// The configure.
/// </summary>
/// <param name="processorManager">
/// The processor manager.
/// </param>
public void Configure(ProcessorManager processorManager)
{
}
