public async Task <int> ComputeNodeReportedAsync([FromBody] ComputeClusterNodeInformation nodeInfo, CancellationToken token)
{
try
{
var nodeName = nodeInfo.Name.ToLowerInvariant();
this.logger.LogInformation("ComputeNodeReported. NodeName {0}, JobCount {1}", nodeName, nodeInfo.Jobs?.Count);
var nodeTable = this.utilities.GetNodesTable();
var jsonTableEntity = new JsonTableEntity(
this.utilities.NodesPartitionKey,
this.utilities.GetHeartbeatKey(nodeName),
nodeInfo);
var result = await nodeTable.ExecuteAsync(TableOperation.InsertOrReplace(jsonTableEntity), null, null, token);
using (HttpResponseMessage r = new HttpResponseMessage((HttpStatusCode)result.HttpStatusCode))
{
r.EnsureSuccessStatusCode();
}
// 30 s
return(this.utilities.Option.HeartbeatIntervalSeconds * 1000);
}
catch (Exception ex)
{
this.logger.LogError(ex, "ComputeNodeReported. NodeName {0}, JobCount {1}", nodeInfo.Name, nodeInfo.Jobs?.Count);
}
return(this.utilities.Option.RetryOnFailureSeconds * 1000);
}
public async Task <int> RegisterRequestedAsync([FromBody] ComputeClusterRegistrationInformation registerInfo, CancellationToken token)
{
try
{
var nodeName = registerInfo.NodeName.ToLowerInvariant();
this.logger.LogInformation("RegisterRequested, NodeName {0}, Distro {1} ", nodeName, registerInfo.DistroInfo);
var nodeTable = this.utilities.GetNodesTable();
var jsonTableEntity = new JsonTableEntity(this.utilities.NodesPartitionKey, this.utilities.GetRegistrationKey(nodeName), registerInfo);
var result = await nodeTable.ExecuteAsync(TableOperation.InsertOrReplace(jsonTableEntity), null, null, token);
using (HttpResponseMessage r = new HttpResponseMessage((HttpStatusCode)result.HttpStatusCode))
{
r.EnsureSuccessStatusCode();
}
// 5 minutes
return(this.utilities.Option.RegistrationIntervalSeconds * 1000);
}
catch (Exception ex)
{
this.logger.LogError(ex, "RegisterRequested. NodeName {0}, Distro {1}",
registerInfo.NodeName, registerInfo.DistroInfo);
}
return(this.utilities.Option.RetryOnFailureSeconds * 1000);
}
public Task CreateParticipentAsync(Guid contestId, CreateParticipentModel participent)
{
var participentEntity = new JsonTableEntity <ParticipentModel>(contestId.ToString(), participent.UserName, new ParticipentModel(contestId, participent.UserName, participent.PushInfo));
var table = this.StorageAccessService.GetTableReference(TableName);
var operation = TableOperation.Insert(participentEntity);
return(table.ExecuteAsync(operation));
}
public async Task <Guid> CreateContestAsync(BeerContestModel contest)
{
Guid contestId = Guid.NewGuid();
contest.Id = contestId;
var entity = new JsonTableEntity <BeerContestModel>(contestId.ToString(), contestId.ToString(), contest);
var table = StorageAccessService.GetTableReference(TableName);
TableOperation operation = TableOperation.Insert(entity);
await table.ExecuteAsync(operation);
return(contestId);
}
public async T.Task RegisterNodeAsync(ComputeClusterRegistrationInformation info, CancellationToken token)
{
var nodeName = info.NodeName.ToLowerInvariant();
this.Logger.Information("RegisterRequested, NodeName {0}, Distro {1} ", nodeName, info.DistroInfo);
var nodeTable = this.Utilities.GetNodesTable();
var jsonTableEntity = new JsonTableEntity(this.Utilities.NodesPartitionKey, this.Utilities.GetRegistrationKey(nodeName), info);
var result = await nodeTable.ExecuteAsync(TableOperation.InsertOrReplace(jsonTableEntity), null, null, token);
using (HttpResponseMessage r = new HttpResponseMessage((HttpStatusCode)result.HttpStatusCode))
{
r.EnsureSuccessStatusCode();
}
}
public async T.Task DoWorkAsync(CancellationToken token)
{
long currentMinute = 0;
while (!token.IsCancellationRequested)
{
try
{
await T.Task.Delay(TimeSpan.FromSeconds(this.workerOptions.MetricsIntervalSeconds), token);
var nodeName = this.ServerOptions.HostName;
// TODO: different frequency
IList <(string, string)> metricScripts = await this.GetMetricScriptsAsync(token);
string toErrorJson(string e) =>
JsonConvert.SerializeObject(new Dictionary <string, string>()
{
{ "Error", e }
}, Formatting.Indented);
var results = await T.Task.WhenAll(metricScripts.Select(async((string, string)s) =>
{
try
{
this.Logger.Debug("Collect metrics for {0}", s.Item1);
var scriptOutput = await PythonExecutor.ExecuteScriptAsync(s.Item2, null, token);
return(s.Item1, scriptOutput.IsError ? toErrorJson(scriptOutput.ErrorMessage) : scriptOutput.Output);
}
catch (Exception ex)
{
return(s.Item1, toErrorJson(ex.ToString()));
}
}));
DynamicTableEntity entity = new DynamicTableEntity(
this.Utilities.MetricsValuesPartitionKey,
nodeName,
"*",
results.ToDictionary(
r => r.Item1,
r => new EntityProperty(r.Item2)));
var result = await metricsTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
var nodesPartitionKey = this.Utilities.GetNodePartitionKey(nodeName);
var time = DateTimeOffset.UtcNow;
var minuteHistoryKey = this.Utilities.GetMinuteHistoryKey();
result = await this.nodesTable.ExecuteAsync(TableOperation.Retrieve <JsonTableEntity>(nodesPartitionKey, minuteHistoryKey), null, null, token);
var history = result.Result is JsonTableEntity historyEntity?historyEntity.GetObject <MetricHistory>() : new MetricHistory(TimeSpan.FromSeconds(10));
var currentMetrics = results.Select(r => new MetricItem()
{
Category = r.Item1,
InstanceValues = JsonConvert.DeserializeObject <Dictionary <string, double?> >(r.Item2)
}).ToList();
history.RangeSeconds = 10;
history.Put(time, currentMetrics);
result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(new JsonTableEntity(nodesPartitionKey, minuteHistoryKey, history)), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
var minute = time.UtcTicks / TimeSpan.TicksPerMinute;
if (minute > currentMinute)
{
currentMinute = minute;
// persist minute data
var currentMetricsEntity = new JsonTableEntity(this.Utilities.GetNodePartitionKey(nodeName),
this.Utilities.GetMinuteHistoryKey(currentMinute),
currentMetrics);
result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(currentMetricsEntity), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
}
}
public override async Task <bool> DoWorkAsync(TaskItem taskItem, CancellationToken token)
{
var job = taskItem.GetMessage <InternalJob>();
var nodeName = this.Configuration.GetValue <string>(Constants.HpcHostNameEnv);
using (this.logger.BeginScope("Do work for InternalJob {0} on node {1}", job.Id, nodeName))
{
// TODO: make sure invisible.
logger.LogInformation("Executing job {0}", job.Id);
var tasks = Enumerable.Range(0, job.CommandLines.Length).Select(async taskId =>
{
var cmd = job.CommandLines[taskId];
logger.LogInformation("Executing command {0}, job {1}", cmd, job.Id);
var taskKey = this.utilities.GetTaskKey(job.Id, taskId, job.RequeueCount);
var resultKey = this.utilities.GetJobResultKey(nodeName, taskKey);
var taskResultBlob = await this.utilities.CreateOrReplaceTaskOutputBlobAsync(job.Id, resultKey, token);
using (var monitor = this.Monitor.StartMonitorTask(taskKey, async(output, cancellationToken) =>
{
try
{
await taskResultBlob.AppendTextAsync(output, Encoding.UTF8, null, null, null, cancellationToken);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Error happened when append to blob {0}", taskResultBlob.Name);
}
}))
{
this.logger.LogInformation("Call startjobandtask for job {0}, task {1}", job.Id, taskKey);
var jobPartitionName = this.utilities.GetJobPartitionKey($"{job.Type}", job.Id);
var nodePartitionName = this.utilities.GetNodePartitionKey(nodeName);
var taskResultArgs = new ComputeNodeTaskCompletionEventArgs(nodeName, job.Id, null)
{
State = TaskState.Dispatching
};
var taskResultEntity = new JsonTableEntity(jobPartitionName, resultKey, taskResultArgs);
var result = await jobsTable.ExecuteAsync(TableOperation.InsertOrReplace(taskResultEntity), null, null, token);
this.logger.LogInformation("Saved task result {0} to jobs table, status code {1}", resultKey, result.HttpStatusCode);
if (!result.IsSuccessfulStatusCode())
{
return(false);
}
var nodeResultEntity = new JsonTableEntity(nodePartitionName, resultKey, taskResultArgs);
result = await nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(nodeResultEntity), null, null, token);
this.logger.LogInformation("Saved task result {0} to nodes table, status code {1}", resultKey, result.HttpStatusCode);
if (!result.IsSuccessfulStatusCode())
{
return(false);
}
await this.communicator.StartJobAndTaskAsync(
nodeName,
new StartJobAndTaskArg(new int[0], job.Id, taskId),
"", "", new ProcessStartInfo(cmd, "", "", $"{this.communicator.Options.AgentUriBase}/output/{taskKey}",
"", new System.Collections.Hashtable(), new long[0], job.RequeueCount), token);
taskResultArgs = new ComputeNodeTaskCompletionEventArgs(nodeName, job.Id, null)
{
State = TaskState.Running
};
taskResultEntity = new JsonTableEntity(jobPartitionName, resultKey, taskResultArgs);
result = await jobsTable.ExecuteAsync(TableOperation.InsertOrReplace(taskResultEntity), null, null, token);
this.logger.LogInformation("Saved task result {0} to jobs table, status code {1}", resultKey, result.HttpStatusCode);
if (!result.IsSuccessfulStatusCode())
{
return(false);
}
nodeResultEntity = new JsonTableEntity(nodePartitionName, resultKey, taskResultArgs);
result = await nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(nodeResultEntity), null, null, token);
this.logger.LogInformation("Saved task result {0} to nodes table, status code {1}", resultKey, result.HttpStatusCode);
if (!result.IsSuccessfulStatusCode())
{
return(false);
}
this.logger.LogInformation("Wait for response for job {0}, task {1}", job.Id, taskKey);
taskResultArgs = await monitor.Execution;
this.logger.LogInformation("Saving result for job {0}, task {1}", job.Id, taskKey);
taskResultArgs.State = TaskState.Finished;
taskResultEntity = new JsonTableEntity(jobPartitionName, resultKey, taskResultArgs);
result = await jobsTable.ExecuteAsync(TableOperation.InsertOrReplace(taskResultEntity), null, null, token);
this.logger.LogInformation("Saved task result {0} to jobs table, status code {1}", resultKey, result.HttpStatusCode);
if (!result.IsSuccessfulStatusCode())
{
return(false);
}
nodeResultEntity = new JsonTableEntity(nodePartitionName, resultKey, taskResultArgs);
result = await nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(nodeResultEntity), null, null, token);
this.logger.LogInformation("Saved task result {0} to nodes table, status code {1}", resultKey, result.HttpStatusCode);
if (!result.IsSuccessfulStatusCode())
{
return(false);
}
return(true);
}
});
var results = await Task.WhenAll <bool>(tasks);
return(results.All(r => r));
}
}
public override async Task <bool> DoWorkAsync(TaskItem taskItem, CancellationToken token)
{
long currentMinute = 0;
while (true)
{
await Task.Delay(this.config.GetValue <int>("MetricInterval"), token);
try
{
var nodeName = this.config.GetValue <string>(Constants.HpcHostNameEnv);
using (this.logger.BeginScope("do metrics loop on {0}", nodeName))
{
// TODO: different frequency
IList <(string, string)> metricScripts = await this.GetMetricScriptsAsync(token);
string toErrorJson(string e) =>
JsonConvert.SerializeObject(new Dictionary <string, string>()
{
{ "Error", e }
}, Formatting.Indented);
var results = await Task.WhenAll(metricScripts.Select(async s =>
{
try
{
this.logger.LogDebug("Collect metrics for {0}", s.Item1);
var psi = new System.Diagnostics.ProcessStartInfo(
@"python",
$"-c \"{s.Item2.Replace("\"", "\\\"")}\"")
{
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true,
};
using (var process = new Process()
{
StartInfo = psi, EnableRaisingEvents = true
})
{
try
{
process.Start();
var output = await process.StandardOutput.ReadToEndAsync();
var error = await process.StandardError.ReadToEndAsync();
return(s.Item1, string.IsNullOrEmpty(error) ? output : toErrorJson(error));
}
finally
{
process.Kill();
}
}
}
catch (Exception ex)
{
return(s.Item1, toErrorJson(ex.ToString()));
}
}));
DynamicTableEntity entity = new DynamicTableEntity(
this.utilities.MetricsValuesPartitionKey,
nodeName,
"*",
results.ToDictionary(
r => r.Item1,
r => new EntityProperty(r.Item2)));
var result = await metricsTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
var nodesPartitionKey = this.utilities.GetNodePartitionKey(nodeName);
var time = DateTimeOffset.UtcNow;
var minuteHistoryKey = this.utilities.GetMinuteHistoryKey();
result = await this.nodesTable.ExecuteAsync(TableOperation.Retrieve <JsonTableEntity>(nodesPartitionKey, minuteHistoryKey), null, null, token);
var history = result.Result is JsonTableEntity historyEntity?historyEntity.GetObject <MetricHistory>() : new MetricHistory(TimeSpan.FromSeconds(10));
var currentMetrics = results.Select(r => new MetricItem()
{
Category = r.Item1,
InstanceValues = JsonConvert.DeserializeObject <Dictionary <string, double?> >(r.Item2)
}).ToList();
history.Range = TimeSpan.FromSeconds(10);
history.Put(time, currentMetrics);
result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(new JsonTableEntity(nodesPartitionKey, minuteHistoryKey, history)), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
var minute = time.UtcTicks / TimeSpan.TicksPerMinute;
if (minute > currentMinute)
{
currentMinute = minute;
// persist minute data
var currentMetricsEntity = new JsonTableEntity(this.utilities.GetNodePartitionKey(nodeName),
this.utilities.GetMinuteHistoryKey(currentMinute),
currentMetrics);
result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(currentMetricsEntity), null, null, token);
if (!result.IsSuccessfulStatusCode())
{
continue;
}
}
}
}
catch (Exception ex)
{
this.logger.LogError(ex, "DoWorkAsync error.");
}
}
}
public async Task UpdateContestEntityAsync(JsonTableEntity <BeerContestModel> entity)
{
var table = StorageAccessService.GetTableReference(TableName);
TableOperation operation = TableOperation.Replace(entity);
await table.ExecuteAsync(operation);
}