/// <summary>
/// Calls the algorithm code (depending on its type).
/// </summary>
static void CallAlgorithm(RunDetails Details, ImageData[] Inputs, ImageData Output)
{
switch (Details.Type)
{
case AlgorithmType.SimpleMap_T: Details.Algorithm.DynamicInvoke(Inputs[0].Data, Output.Data, Details.Parameters[0]); break;
case AlgorithmType.SimpleMap_TU: Details.Algorithm.DynamicInvoke(Inputs[0].Data, Output.Data, Details.Parameters[0], Details.Parameters[1]); break;
case AlgorithmType.SimpleMap_TUV: Details.Algorithm.DynamicInvoke(Inputs[0].Data, Output.Data, Details.Parameters[0], Details.Parameters[1], Details.Parameters[2]); break;
case AlgorithmType.PositionMap:
ImageSegmentPosition pmIP = GetPosition(Inputs[0]);
ImageSegmentPosition pmOP = GetPosition(Output);
Details.Algorithm.DynamicInvoke(Inputs[0].Data, Output.Data, pmIP, pmOP, Details.Parameters[0]);
break;
case AlgorithmType.Combiner:
ImageSegmentPosition[] cIP = Inputs.Select(GetPosition).ToArray();
ImageSegmentPosition cOP = GetPosition(Output);
Details.Algorithm.DynamicInvoke(Inputs.Select((x) => x.Data).ToArray(), Output.Data, cIP, cOP, Details.Parameters[0]); break;
case AlgorithmType.Extractor: Details.Algorithm.DynamicInvoke(Inputs[0].Data, Details.Parameters[0]); break;
case AlgorithmType.PositionExtractor: Details.Algorithm.DynamicInvoke(Inputs[0].Data, GetPosition(Inputs[0]), Details.Parameters[0]); break;
}
}
/// <summary>
/// Prepares algorithm geometry.
/// </summary>
/// <param name="Details">Parameters to prepare.</param>
/// <param name="Parameters">Input parameters.</param>
internal static void PrepareGeometry(ref RunDetails Details, AlgorithmRunParameters Parameters)
{
/* Copies common parameters */
Details.FillZero = Parameters.FillZero;
Details.Xstep = Parameters.Xstep;
if (Details.Xstep == 0)
{
if (Details.OutputImage != null)
{
Details.Xstep = (int)Details.OutputImage.Width;
}
else
{
Details.Xstep = (int)Details.InputImages[0].Width;
}
}
Details.Ystep = Parameters.Ystep;
Details.InputMargins = Parameters.InputMargins;
/* Compute block sizes */
if (Details.OutputImage != null)
{
Details.DataHeight = (int)Details.OutputImage.Height; Details.DataWidth = (int)Details.OutputImage.Width;
}
else
{
Details.DataHeight = (int)Details.InputImages[0].Height; Details.DataWidth = (int)Details.InputImages[0].Width;
}
}
private static void SaveCurrentState()
{
#if !NETSTANDARD1_3
var newDetails = new RunDetails
{
Retrievals = _retrievalCount,
Submissions = _submissionCount,
SessionExpiry = DateTime.Now.AddHours(1)
};
var json = Serializer.Serialize(newDetails);
if (_holdPersistence)
{
return;
}
lock (CountsLock)
{
if (_holdPersistence)
{
return;
}
try
{
Environment.SetEnvironmentVariable(LicenseEnvironmentVariable, json.ToString(), EnvironmentVariableTarget.User);
}
catch (Exception e)
{
// nothing else we can really do here
TrelloConfiguration.Log.Error(e);
}
}
#endif
}
public static void Scheduler(RunDetails Details)
{
int Parallelism = Environment.ProcessorCount;
ThreadDetails[] thDetails = new ThreadDetails[Parallelism];
/* Compute block sizes. Make sure block sizes are multiples of Ystep (except for last block). */
int StepSize = (Details.DataHeight + Parallelism - 1) / Parallelism;
if (Details.Ystep != 0)
{
StepSize = (StepSize + Details.Ystep - 1) / Details.Ystep * Details.Ystep;
}
/* Update blocks */
for (int i = 0; i < Parallelism; i++)
{
thDetails[i] = new ThreadDetails()
{
CurrentPositionX = 0, CurrentPositionY = i * StepSize, StartPosition = i * StepSize, EndPosition = (i + 1) * StepSize
}
}
;
for (int i = 0; i < Parallelism; i++)
{
if (thDetails[i].EndPosition > Details.DataHeight)
{
thDetails[i].EndPosition = Details.DataHeight;
}
}
/* Run in parallel */
Parallel.For(0, Parallelism, (i) => ProcessBlock(Details, ref thDetails[i]));
}
static void LockDataNofill(RunDetails RD, ThreadDetails TD, Image Image, ref ImageData Data, bool Readonly)
{
bool NotInitialized = TD.CurrentPositionY == TD.StartPosition && TD.CurrentPositionX == 0;
/* Compute required height and width */
int NWidth = RD.Xstep;
int NHeight = RD.Ystep;
if (TD.CurrentPositionX + RD.Xstep > Image.Width)
{
NWidth = (int)Image.Width - TD.CurrentPositionX;
}
if (TD.CurrentPositionY + RD.Ystep > TD.EndPosition)
{
NHeight = TD.EndPosition - TD.CurrentPositionY;
}
/* If window size must be changed */
if (NWidth != Data.Position.Width || NHeight != Data.Position.Height || NotInitialized)
{
if (!NotInitialized)
{
Image.ExitLock(Data);
}
Data = Image.LockData(new System.Drawing.Rectangle(TD.CurrentPositionX, TD.CurrentPositionY, NWidth, NHeight), false, Readonly);
}
/* Just swap otherwise */
else
{
Data = Image.SwitchLockData(Data, TD.CurrentPositionX, TD.CurrentPositionY, false, Readonly);
}
}
public void Visit(RunDetails details, IResultLogger[] loggers)
{
foreach (StoreTaskDetails storeDetail in details.StoreDetails)
{
Write(loggers, logger => logger.LogStore(storeDetail));
VisitCosts(storeDetail, storeDetail.Cost, 1, loggers);
}
}
public void ReportFailTrial(TrialSettings settings, Exception exp)
{
var result = new BinaryClassificationTrialResult
{
TrialSettings = settings,
Exception = exp,
};
RunDetails.Add(result);
}
private void SaveRun_click(object sender, RoutedEventArgs e)
{
var newRunDetails = new RunDetails();
var item = (RunDetails)(sender as FrameworkElement)?.DataContext;
if (item != null)
{
}
}
private void SaveRun_click(object sender, RoutedEventArgs e)
{
var newRunDetails = new RunDetails();
var item = (RunDetails) (sender as FrameworkElement)?.DataContext;
if (item != null)
{
}
}
public void ReportCompletedTrial(TrialResult result)
{
if (result is BinaryClassificationTrialResult binaryClassificationResult)
{
RunDetails.Add(binaryClassificationResult);
OnTrialCompleted?.Invoke(this, binaryClassificationResult);
}
else
{
throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}");
}
}
public static void SaveCurrentState()
{
var newDetails = new RunDetails
{
Retrievals = _retrievalCount,
Submissions = _submissionCount,
SessionExpiry = DateTime.Now.AddHours(1)
};
var json = Serializer.Serialize(newDetails);
File.WriteAllText(DetailsPath, json.ToString());
}
protected override void ProcessRecord()
{
if (this.Count > 0)
{
this.WriteObject(this.MAInstance.GetRunHistory(this.Count), true);
}
else
{
using (RunDetails lastRun = this.MAInstance.GetLastRun())
{
this.WriteObject(lastRun, true);
}
}
}
public Task <RunDetails> RunForEachEnabledStore(IStoreCommand command, CancellationToken cancellationToken)
{
RunDetails result = new RunDetails();
foreach (IStore enabledStore in EnabledStores)
{
EnsureStoreInitialised(enabledStore).Wait(cancellationToken);
var storeTaskDetails = Run(result, command, enabledStore, cancellationToken);
result.StoreDetails.Add(storeTaskDetails);
}
return(Task.WhenAll(result.Tasks).ContinueWith <RunDetails>((Func <Task, RunDetails>)(t => result), cancellationToken));
}
/// <summary>
/// Runs the given algorithm on the input data.
/// </summary>
/// <typeparam name="T">First extra parameter type.</typeparam>
/// <typeparam name="U">Second extra parameter type.</typeparam>
/// <typeparam name="V">Third extra parameter type.</typeparam>
/// <param name="Algorithm">Parallel algorithm.</param>
/// <param name="Argument1">First argument to be passed to all invocations.</param>
/// <param name="Argument2">Second argument to be passed to all invocations.</param>
/// <param name="Argument3">Third argument to be passed to all invocations.</param>
/// <param name="Input">Input image.</param>
/// <param name="Output">Output image.</param>
/// <param name="Parameters">Parameters of the algorithm run.</param>
public static void RunAlgorithm <T, U, V>(SimpleMap <T, U, V> Algorithm, T Argument1, U Argument2, V Argument3, Image Input, Image Output, AlgorithmRunParameters Parameters)
{
RunDetails details = new RunDetails()
{
Algorithm = Algorithm,
InputImages = new Image[] { Input },
OutputImage = Output,
Parameters = new object[] { Argument1, Argument2, Argument3 },
Type = AlgorithmType.SimpleMap_TUV,
};
PrepareGeometry(ref details, Parameters);
DefaultScheduler(details);
}
/// <summary>
/// Runs the given algorithm on the input data.
/// </summary>
/// <typeparam name="T">Extra parameter type.</typeparam>
/// <param name="Algorithm">Parallel algorithm.</param>
/// <param name="Argument">Argument to be passed to all invocations.</param>
/// <param name="Input">Input image.</param>
/// <param name="Parameters">Parameters of the algorithm run.</param>
public static void RunAlgorithm <T>(Extractor <T> Algorithm, T Argument, Image Input, AlgorithmRunParameters Parameters)
{
RunDetails details = new RunDetails()
{
Algorithm = Algorithm,
InputImages = new Image[] { Input },
OutputImage = null,
Parameters = new object[] { Argument },
Type = AlgorithmType.Extractor,
};
PrepareGeometry(ref details, Parameters);
DefaultScheduler(details);
}
/// <summary>
/// Runs the given algorithm on the input data.
/// </summary>
/// <typeparam name="T">Extra parameter type.</typeparam>
/// <param name="Algorithm">Parallel algorithm.</param>
/// <param name="Argument">Argument to be passed to all invocations.</param>
/// <param name="Inputs">Input images.</param>
/// <param name="Output">Output image.</param>
/// <param name="Parameters">Parameters of the algorithm run.</param>
public static void RunAlgorithm <T>(Combiner <T> Algorithm, T Argument, Image[] Inputs, Image Output, AlgorithmRunParameters Parameters)
{
RunDetails details = new RunDetails()
{
Algorithm = Algorithm,
InputImages = Inputs,
OutputImage = Output,
Parameters = new object[] { Argument },
Type = AlgorithmType.Combiner,
};
PrepareGeometry(ref details, Parameters);
DefaultScheduler(details);
}
/// <summary>
/// Runs the given algorithm on the input data.
/// </summary>
/// <typeparam name="T">Extra parameter type.</typeparam>
/// <param name="Algorithm">Parallel algorithm.</param>
/// <param name="Argument">Argument to be passed to all invocations.</param>
/// <param name="Input">Input image.</param>
/// <param name="Output">Output image.</param>
/// <param name="Parameters">Parameters of the algorithm run.</param>
public static void RunAlgorithm <T>(PositionDependentMap <T> Algorithm, T Argument, Image Input, Image Output, AlgorithmRunParameters Parameters)
{
RunDetails details = new RunDetails()
{
Algorithm = Algorithm,
InputImages = new Image[] { Input },
OutputImage = Output,
Parameters = new object[] { Argument },
Type = AlgorithmType.PositionMap,
};
PrepareGeometry(ref details, Parameters);
DefaultScheduler(details);
}
public static string GetMessageBody(RunDetails r, string thresholdMessage)
{
StringBuilder builder = new StringBuilder();
if (thresholdMessage != null)
{
builder.AppendFormat(MessageBuilder.GetTemplate("ThresholdExceededFragment"), r.RunProfileName, r.MAName, r.StartTime, r.EndTime, r.SecurityID, r.LastStepStatus, r.RunNumber, thresholdMessage);
}
else
{
builder.AppendFormat(MessageBuilder.GetTemplate("RunSummaryFragment"), r.RunProfileName, r.MAName, r.StartTime, r.EndTime, r.SecurityID, r.LastStepStatus, r.RunNumber);
}
builder.AppendFormat(MessageBuilder.GetTemplate("StepTableFragment"), MessageBuilder.BuildStepDetails(r.StepDetails));
string stagingErrors = MessageBuilder.BuildStagingErrorDetails(r.StepDetails);
if (stagingErrors != null)
{
builder.AppendLine(stagingErrors);
}
string importErrors = MessageBuilder.BuildImportErrorDetails(r.StepDetails);
if (importErrors != null)
{
builder.AppendLine(importErrors);
}
string exportErrors = MessageBuilder.BuildExportErrorDetails(r.StepDetails);
if (exportErrors != null)
{
builder.AppendLine(exportErrors);
}
string syncErrors = MessageBuilder.BuildSyncErrorDetails(r.StepDetails);
if (syncErrors != null)
{
builder.AppendLine(syncErrors);
}
InlineResult result = PreMailer.Net.PreMailer.MoveCssInline(MessageBuilder.GetTemplate("EmailTemplate").Replace("%BODY%", builder.ToString()));
return(result.Html);
}
protected override void ProcessRecord()
{
RunDetails last = this.MAInstance.GetLastRun();
while (!this.MAInstance.IsIdle())
{
this.UpdateProgress(true, last.RunNumber);
System.Threading.Thread.Sleep(5000);
}
ProgressRecord r = new ProgressRecord(0, this.MAInstance.Name, string.Format("Finished: {0}", last.RunProfileName));
r.RecordType = ProgressRecordType.Completed;
r.PercentComplete = 100;
this.WriteProgress(r);
}
protected override void ProcessRecord()
{
if (this.Item != null)
{
using (RunDetails syncrundetail = SyncServer.GetRunDetail(this.Item))
{
this.WriteObject(syncrundetail);
}
}
else
{
using (RunDetails marundetail = this.MAInstance.GetRunDetail(this.RunNumber))
{
this.WriteObject(marundetail);
}
}
}
/// <summary>
/// Process a block of data.
/// </summary>
/// <param name="RunDetails">Thread-common run parameters.</param>
/// <param name="ThDetails">Thread-specific run parameters.</param>
static void ProcessBlock(RunDetails RunDetails, ref ThreadDetails ThDetails)
{
/* Initialized inputs and outputs */
ImageData[] Dataset = new ImageData[RunDetails.InputImages.Length];
ImageData OutputData = default(ImageData);
/* Check for cases where split produces no data for current CPU. */
if (ThDetails.CurrentPositionY > ThDetails.EndPosition)
{
return;
}
/* While there is data to process */
while (ThDetails.CurrentPositionY < ThDetails.EndPosition)
{
/* Read data and initialize output */
for (int i = 0; i < Dataset.Length; i++)
{
ReadImageBlock(RunDetails, RunDetails.InputImages[i], ref Dataset[i], ref ThDetails);
}
if (RunDetails.OutputImage != null)
{
ProcessOutput(RunDetails, ThDetails, ref OutputData);
}
CallAlgorithm(RunDetails, Dataset, OutputData);
ThDetails.CurrentPositionX += RunDetails.Xstep;
if (ThDetails.CurrentPositionX >= RunDetails.DataWidth)
{
ThDetails.CurrentPositionX = 0; ThDetails.CurrentPositionY += RunDetails.Ystep;
}
}
/* Release inputs and outputs */
for (int i = 0; i < Dataset.Length; i++)
{
RunDetails.InputImages[i].ExitLock(Dataset[i]);
}
if (RunDetails.OutputImage != null)
{
RunDetails.OutputImage.ExitLock(OutputData);
}
}
/// <summary>
/// Reads a block of data from the input images.
/// </summary>
public static void ReadImageBlock(RunDetails RD, Image Selected, ref ImageData Data, ref ThreadDetails TD)
{
if (!RD.FillZero)
{
LockDataNofill(RD, TD, Selected, ref Data, true); return;
}
if (TD.CurrentPositionY == TD.StartPosition && TD.CurrentPositionX == 0)
{
Data = Selected.LockData(new System.Drawing.Rectangle(
TD.CurrentPositionX - RD.InputMargins, TD.CurrentPositionY - RD.InputMargins,
RD.Xstep + 2 * RD.InputMargins, RD.Ystep + 2 * RD.InputMargins),
RD.FillZero);
}
else
{
Data = Selected.SwitchLockData(Data, TD.CurrentPositionX - RD.InputMargins, TD.CurrentPositionY - RD.InputMargins, RD.FillZero);
}
}
public StoreTaskDetails Run(RunDetails runDetails, IStoreCommand command, IStore store, CancellationToken cancellationToken)
{
Stopwatch sw = new Stopwatch();
sw.Start();
Task <GroupOperationCost> task = command.DoAsync(store, cancellationToken);
StoreTaskDetails taskDetails = new StoreTaskDetails(command, store, task);
task.ContinueWith(t =>
{
sw.Stop();
taskDetails.Elapsed = sw.Elapsed;
if (task.IsCompleted)
{
taskDetails.Cost = task.Result;
}
}, cancellationToken);
return(taskDetails);
}
public void ExecutionComplete(RunDetails d)
{
if (this.powershell == null || !this.SupportsExecutionComplete)
{
return;
}
this.powershell.ResetState();
this.powershell.AddCommand("ExecutionComplete");
this.powershell.AddArgument(d);
try
{
this.powershell.Invoke();
this.powershell.ThrowOnPipelineError();
}
catch (RuntimeException ex)
{
if (ex.InnerException is UnexpectedChangeException)
{
this.HasStoppedMA = true;
throw ex.InnerException;
}
else
{
logger.Error(ex, $"{this.config.ManagementAgentName}: ExecutionComplete handler threw an exception");
}
}
catch (UnexpectedChangeException)
{
this.HasStoppedMA = true;
throw;
}
catch (Exception ex)
{
logger.Error(ex, $"{this.config.ManagementAgentName}: ExecutionComplete handler threw an exception");
}
}
public ThresholdExceededException(string message, RunDetails r, Exception innerException)
: base(message, innerException)
{
this.RunDetails = r;
}
public ThresholdExceededException(string message, RunDetails r)
: base(message)
{
this.RunDetails = r;
}
protected override void ProcessRecord()
{
RunDetails last = this.MAInstance.GetLastRun();
int lastExecutionNumber = last?.RunNumber ?? -1;
while (!this.MAInstance.IsIdle())
{
if (!this.NoProgress.IsPresent)
{
this.UpdateProgress(true, lastExecutionNumber);
}
Thread.Sleep(5000);
}
if (this.Async)
{
this.MAInstance.ExecuteRunProfileAsync(this.RunProfileName, this.ResumeLastRun);
}
else
{
Task <string> t = this.MAInstance.ExecuteRunProfileAsync(this.RunProfileName, this.ResumeLastRun);
int currentExecutionNumber = -1;
do
{
if (t.IsCompleted)
{
break;
}
RunDetails d = this.MAInstance.GetLastRun();
if (d != null)
{
currentExecutionNumber = d.RunNumber;
}
Thread.Sleep(500);
}while (currentExecutionNumber == -1 || currentExecutionNumber == lastExecutionNumber);
while (!t.IsCompleted)
{
if (!this.NoProgress.IsPresent)
{
this.UpdateProgress(false, currentExecutionNumber);
}
Thread.Sleep(2000);
}
ProgressRecord r = new ProgressRecord(0, this.MAInstance.Name, $"Finished: {this.RunProfileName}")
{
RecordType = ProgressRecordType.Completed,
PercentComplete = 100
};
this.WriteProgress(r);
if (t.IsFaulted)
{
throw t.Exception?.InnerExceptions.First() ?? new MAExecutionException();
}
if (t.Result != "success")
{
this.WriteWarning($"Management agent returned {t.Result}");
}
}
}
protected void UpdateProgress(bool pending, int runNumber)
{
if (runNumber < 0)
{
return;
}
RunDetails s = this.MAInstance.GetRunDetail(runNumber);
if (s == null)
{
return;
}
if (this.runProfile == null)
{
this.runProfile = this.MAInstance.RunProfiles[s.RunProfileName];
this.stats = this.MAInstance.Statistics;
}
StepDetails d = s.StepDetails.FirstOrDefault();
if (d == null)
{
return;
}
string description;
if (pending)
{
description = $"Waiting for {this.MAInstance.Name} to finish {s.RunProfileName}";
}
else
{
description = this.MAInstance.Name;
}
if (this.lastStepNumber == 0 || this.lastStepNumber != d.StepNumber)
{
this.lastStepNumber = d.StepNumber;
this.countHistory = new FixedSizedQueue <ProgressItem>(30);
}
ProgressRecord r = new ProgressRecord(0, description, string.Format(
$"Performing {this.runProfile.Name} step {d.StepNumber}/{this.runProfile.RunSteps.Count}: {d.StepDefinition.StepTypeDescription}"))
{
RecordType = ProgressRecordType.Processing
};
int processed;
double total;
int remaining = 0;
if (this.GetCounts(d, out processed, out total))
{
int percentComplete = (int)((processed / total) * 100);
r.PercentComplete = percentComplete > 100 ? 0 : percentComplete;
remaining = (int)total - processed;
}
if (processed > 0)
{
double objpersec = 0;
int changedCount;
TimeSpan?timespan;
this.GetCountDiff(processed, out changedCount, out timespan);
if (changedCount > 0 && timespan.HasValue)
{
objpersec = changedCount / timespan.Value.TotalSeconds;
}
if (remaining > 0 && objpersec > 0)
{
int remainingSeconds = (int)(remaining / objpersec);
r.SecondsRemaining = remainingSeconds > 0 ? remainingSeconds : 0;
}
if (objpersec > 0 && !double.IsInfinity(objpersec))
{
r.StatusDescription += $" ({objpersec:N2} obj/sec)";
}
}
else
{
r.StatusDescription += " (waiting for MA to start)";
}
this.WriteProgress(r);
}
private void SaveDetails(RunDetails details)
{
}
/// <summary>
/// Initializes and writes data to output image.
/// </summary>
public static void ProcessOutput(RunDetails RunDetails, ThreadDetails ThDetails, ref ImageData OutputData)
{
LockDataNofill(RunDetails, ThDetails, RunDetails.OutputImage, ref OutputData, false);
}
protected override void ProcessRecord()
{
this.WriteObject(RunDetails.LoadRunDetails(this.File), true);
}
public static string GetMessageBody(RunDetails r)
{
return(GetMessageBody(r, null));
}
