public async T.Task <DispatchResult> DispatchAsync(Job job, CancellationToken token)
{
// TODO: github integration
var jobTable = this.Utilities.GetJobsTable();
this.Logger.Information("Generating tasks for job {0}", job.Id);
var diagTest = await jobTable.RetrieveAsync <InternalDiagnosticsTest>(this.Utilities.GetDiagPartitionKey(job.DiagnosticTest.Category), job.DiagnosticTest.Name, token);
if (diagTest != null)
{
var scriptBlob = this.Utilities.GetBlob(diagTest.DispatchScript.ContainerName, diagTest.DispatchScript.Name);
var nodesTable = this.Utilities.GetNodesTable();
var metadataKey = this.Utilities.GetMetadataKey();
this.Logger.Information("GenerateTasks, Querying node info for job {0}", job.Id);
var metadata = await T.Task.WhenAll(job.TargetNodes.Select(async n =>
{
var heartbeatKey = this.Utilities.GetHeartbeatKey(n);
var nodeInfo = await nodesTable.RetrieveJsonTableEntityAsync(this.Utilities.NodesPartitionKey, heartbeatKey, token);
return(new
{
Node = n,
Metadata = await nodesTable.RetrieveAsync <object>(this.Utilities.GetNodePartitionKey(n), metadataKey, token),
Heartbeat = nodeInfo?.GetObject <ComputeClusterNodeInformation>(),
LastHeartbeatTime = nodeInfo?.Timestamp,
NodeRegistrationInfo = await nodesTable.RetrieveAsync <ComputeClusterRegistrationInformation>(this.Utilities.NodesPartitionKey, this.Utilities.GetRegistrationKey(n), token),
});
}));
var dispatchTasks = await PythonExecutor.ExecuteAsync(scriptBlob, new { Job = job, Nodes = metadata }, token);
if (dispatchTasks.IsError)
{
this.Logger.Error("Generate tasks, job {0}, {1}", job.Id, dispatchTasks.ErrorMessage);
await this.Utilities.FailJobWithEventAsync(
job,
$"Dispatch script {dispatchTasks.ErrorMessage}",
token);
return(null);
}
else
{
return(JsonConvert.DeserializeObject <DispatchResult>(dispatchTasks.Output));
}
}
else
{
await this.Utilities.FailJobWithEventAsync(
job,
$"No diag test {job.DiagnosticTest.Category}/{job.DiagnosticTest.Name} found",
token);
this.Logger.Error("No diag test found");
return(null);
}
}
private void runPython()
{
PipeClient client;
python = new PythonExecutor();
if (!python.CanRun)
{
if (System.IO.File.Exists("Python35\\python.exe"))
{
python = new PythonExecutor("Python35\\python.exe");
}
else if (MessageBox.Show("Unable to find python.\nDo you want to download an example interpreter from cheonghyun.com?", "IpcPythonCS", MessageBoxButtons.YesNo, MessageBoxIcon.Question)
== System.Windows.Forms.DialogResult.Yes)
{
PythonDownloader down = new PythonDownloader();
down.DownloadAndUnzip();
python = new PythonExecutor(down.PythonInterpreter.FullName);
}
else
{
MessageBox.Show("This application cannot run without Python.", "Error", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}
}
python.RunScript("main.py");
client = new PipeClient();
client.Connect("calculator");
calculator = new PyCalculator(client);
}
private void Init()
{
isFirstLoad = true;
Predictions = null;
python = !string.IsNullOrEmpty(pythonPath) ? new PythonExecutor(pythonPath) : new PythonExecutor();
python.AddStandartOutputErrorFilters("Using TensorFlow backend.");
python.AddStandartOutputErrorFilters("CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected");
client = new PipeClient();
predict = new Predict(client);
python.OnPythonError += Python_OnPythonError;
TinfoBox.Start(initText, title);
python.RunScript("main.py");
client.Connect("openstsm");
}
public async T.Task <string> AggregateTasksAsync(Job job, List <Task> tasks, List <ComputeClusterTaskInformation> taskResults, CancellationToken token)
{
var jobTable = this.Utilities.GetJobsTable();
this.Logger.Information("AggregateTasks, job {0}", job.Id);
var diagTest = await jobTable.RetrieveAsync <InternalDiagnosticsTest>(
this.Utilities.GetDiagPartitionKey(job.DiagnosticTest.Category),
job.DiagnosticTest.Name,
token);
string result = string.Empty;
if (diagTest != null)
{
var scriptBlob = this.Utilities.GetBlob(diagTest.AggregationScript.ContainerName, diagTest.AggregationScript.Name);
var aggregationResult = await PythonExecutor.ExecuteAsync(scriptBlob, new { Job = job, Tasks = tasks, TaskResults = taskResults }, token);
result = aggregationResult.Output;
if (aggregationResult.IsError)
{
this.Logger.Error("AggregateTasks, job {0}, {1}", job.Id, aggregationResult.ErrorMessage);
(job.Events ?? (job.Events = new List <Event>())).Add(new Event()
{
Content = $"Diag reduce script {aggregationResult.ErrorMessage}",
Source = EventSource.Job,
Type = EventType.Alert,
});
job.State = JobState.Failed;
}
}
else
{
job.State = JobState.Failed;
(job.Events ?? (job.Events = new List <Event>())).Add(new Event()
{
Content = $"No diag test {job.DiagnosticTest.Category}/{job.DiagnosticTest.Name} found",
Source = EventSource.Job,
Type = EventType.Alert
});
}
return(result);
}
private async T.Task <bool> TaskResultHook(
Job job,
int taskId,
string path,
CancellationToken token)
{
var jobPartitionKey = this.Utilities.GetJobPartitionKey(job.Type, job.Id);
var taskResultKey = this.Utilities.GetTaskResultKey(job.Id, taskId, job.RequeueCount);
var taskInfo = await this.jobsTable.RetrieveAsync <ComputeClusterTaskInformation>(jobPartitionKey, taskResultKey, token);
if (taskInfo == null)
{
this.Logger.Information("There is no task info recorded for job {0}, task {1}, requeue {2}, skip the filter.", job.Id, taskId, job.RequeueCount);
return(true);
}
var filteredResult = await PythonExecutor.ExecuteAsync(path, new { Job = job, Task = taskInfo }, token);
this.Logger.Information("Task filter script execution for job {0}, task {1}, filteredResult exit code {2}, result length {3}", job.Id, taskId, filteredResult.ExitCode, filteredResult.Output?.Length);
taskInfo.Message = filteredResult.IsError ? taskInfo.Message : filteredResult.Output;
await this.jobsTable.InsertOrReplaceAsync(jobPartitionKey, taskResultKey, taskInfo, token);
if (filteredResult.IsError)
{
this.Logger.Error("There is an error in task filter script. {0}", filteredResult.ErrorMessage);
await this.Utilities.UpdateJobAsync(job.Type, job.Id, j =>
{
(j.Events ?? (j.Events = new List <Event>())).Add(new Event()
{
Content = filteredResult.ErrorMessage,
Source = EventSource.Job,
Type = EventType.Alert,
});
j.State = JobState.Failed;
}, token, this.Logger);
return(true);
}
return(true);
}
public void RunSelectiveSearch()
{
Predict predict;
PythonExecutor python = new PythonExecutor(@"C:\Users\kutiatore\AppData\Local\Programs\Python\Python35\python.exe");
python.AddStandartOutputErrorFilters("Using TensorFlow backend.");
python.AddStandartOutputErrorFilters("CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected");
PipeClient client = new PipeClient();
predict = new Predict(client);
try
{
python.RunScript("main.py");
client.Connect("openstsm");
predict = new Predict(client);
bool run = predict.LoadModel(@"E:\\Storage\\Python\\OpenSTSM\\ML\\models\\model.model");
run = predict.ImageDimessionCorrections("E:\\Libraries\\Desktop\\Visa Docs\\test_selective_1.png");
run = predict.RunSelectiveSearch(80, 1.0f);
string results = predict.RunPrediction(5, 3, 5, 1, 8, 2, true);
client.Close();
python.Close();
}
catch (System.Exception e)
{
System.Diagnostics.Debug.WriteLine("---------------------");
System.Diagnostics.Debug.WriteLine(e.StackTrace);
System.Diagnostics.Debug.WriteLine("---------------------");
if (client.isConnected())
{
client.Close();
}
python.Close();
}
}
public async T.Task DoWorkAsync(CancellationToken token)
{
long currentMinute = 0;
while (!token.IsCancellationRequested)
{
try
{
await T.Task.Delay(TimeSpan.FromSeconds(this.workerOptions.MetricsIntervalSeconds), token);
var nodeName = this.ServerOptions.HostName;
// TODO: different frequency
IList <(string, string)> metricScripts = await this.GetMetricScriptsAsync(token);
string toErrorJson(string e) =>
JsonConvert.SerializeObject(new Dictionary <string, string>()
{
{ "Error", e }
}, Formatting.Indented);
var results = await T.Task.WhenAll(metricScripts.Select(async((string, string)s) =>
{
try
{
this.Logger.Debug("Collect metrics for {0}", s.Item1);
var scriptOutput = await PythonExecutor.ExecuteScriptAsync(s.Item2, null, token);
return(s.Item1, scriptOutput.IsError ? toErrorJson(scriptOutput.ErrorMessage) : scriptOutput.Output);
}
catch (Exception ex)
{
return(s.Item1, toErrorJson(ex.ToString()));
}
}));
DynamicTableEntity entity = new DynamicTableEntity(
this.Utilities.MetricsValuesPartitionKey,
nodeName,
"*",
results.ToDictionary(
r => r.Item1,
r => new EntityProperty(r.Item2)));
var result = await metricsTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
var nodesPartitionKey = this.Utilities.GetNodePartitionKey(nodeName);
var time = DateTimeOffset.UtcNow;
var minuteHistoryKey = this.Utilities.GetMinuteHistoryKey();
result = await this.nodesTable.ExecuteAsync(TableOperation.Retrieve <JsonTableEntity>(nodesPartitionKey, minuteHistoryKey), null, null, token);
var history = result.Result is JsonTableEntity historyEntity?historyEntity.GetObject <MetricHistory>() : new MetricHistory(TimeSpan.FromSeconds(10));
var currentMetrics = results.Select(r => new MetricItem()
{
Category = r.Item1,
InstanceValues = JsonConvert.DeserializeObject <Dictionary <string, double?> >(r.Item2)
}).ToList();
history.RangeSeconds = 10;
history.Put(time, currentMetrics);
result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(new JsonTableEntity(nodesPartitionKey, minuteHistoryKey, history)), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
var minute = time.UtcTicks / TimeSpan.TicksPerMinute;
if (minute > currentMinute)
{
currentMinute = minute;
// persist minute data
var currentMetricsEntity = new JsonTableEntity(this.Utilities.GetNodePartitionKey(nodeName),
this.Utilities.GetMinuteHistoryKey(currentMinute),
currentMetrics);
result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(currentMetricsEntity), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
}
}
public async T.Task <List <InternalTask> > GenerateTasksAsync(Job job, CancellationToken token)
{
// TODO: github integration
var jobTable = this.Utilities.GetJobsTable();
this.Logger.Information("Generating tasks for job {0}", job.Id);
var diagTest = await jobTable.RetrieveAsync <InternalDiagnosticsTest>(this.Utilities.GetDiagPartitionKey(job.DiagnosticTest.Category), job.DiagnosticTest.Name, token);
if (diagTest != null)
{
var scriptBlob = this.Utilities.GetBlob(diagTest.DispatchScript.ContainerName, diagTest.DispatchScript.Name);
var nodesTable = this.Utilities.GetNodesTable();
var metadataKey = this.Utilities.GetMetadataKey();
this.Logger.Information("GenerateTasks, Querying node info for job {0}", job.Id);
var metadata = await T.Task.WhenAll(job.TargetNodes.Select(async n => new
{
Node = n,
Metadata = await nodesTable.RetrieveAsync <object>(this.Utilities.GetNodePartitionKey(n), metadataKey, token),
NodeRegistrationInfo = await nodesTable.RetrieveAsync <ComputeClusterRegistrationInformation>(this.Utilities.NodesPartitionKey, this.Utilities.GetRegistrationKey(n), token),
}));
var dispatchTasks = await PythonExecutor.ExecuteAsync(scriptBlob, new { Job = job, Nodes = metadata }, token);
if (dispatchTasks.IsError)
{
this.Logger.Error("Generate tasks, job {0}, {1}", job.Id, dispatchTasks.ErrorMessage);
await this.Utilities.UpdateJobAsync(job.Type, job.Id, j =>
{
j.State = JobState.Failed;
(j.Events ?? (j.Events = new List <Event>())).Add(new Event()
{
Content = $"Dispatch script {dispatchTasks.ErrorMessage}",
Source = EventSource.Job,
Type = EventType.Alert
});
}, token, this.Logger);
return(null);
}
else
{
return(JsonConvert.DeserializeObject <List <InternalTask> >(dispatchTasks.Output));
}
}
else
{
await this.Utilities.UpdateJobAsync(job.Type, job.Id, j =>
{
j.State = JobState.Failed;
(j.Events ?? (j.Events = new List <Event>())).Add(new Event()
{
Content = $"No diag test {job.DiagnosticTest.Category}/{job.DiagnosticTest.Name} found",
Source = EventSource.Job,
Type = EventType.Alert
});
}, token, this.Logger);
this.Logger.Error("No diag test found");
return(null);
}
}
protected string[] RunMLModel()
{
FileManager.WriteToFile(pythonIn, MLData, ",");
string[] output = PythonExecutor.RunExecutor(AI_PYTHON_LOC + pythonProgName, Application.persistentDataPath + pythonIn);
return(output);
}