public async Task <int> ComputeNodeReportedAsync([FromBody] ComputeClusterNodeInformation nodeInfo, CancellationToken token) { try { var nodeName = nodeInfo.Name.ToLowerInvariant(); this.logger.LogInformation("ComputeNodeReported. NodeName {0}, JobCount {1}", nodeName, nodeInfo.Jobs?.Count); var nodeTable = this.utilities.GetNodesTable(); var jsonTableEntity = new JsonTableEntity( this.utilities.NodesPartitionKey, this.utilities.GetHeartbeatKey(nodeName), nodeInfo); var result = await nodeTable.ExecuteAsync(TableOperation.InsertOrReplace(jsonTableEntity), null, null, token); using (HttpResponseMessage r = new HttpResponseMessage((HttpStatusCode)result.HttpStatusCode)) { r.EnsureSuccessStatusCode(); } // 30 s return(this.utilities.Option.HeartbeatIntervalSeconds * 1000); } catch (Exception ex) { this.logger.LogError(ex, "ComputeNodeReported. NodeName {0}, JobCount {1}", nodeInfo.Name, nodeInfo.Jobs?.Count); } return(this.utilities.Option.RetryOnFailureSeconds * 1000); }
public async Task <int> RegisterRequestedAsync([FromBody] ComputeClusterRegistrationInformation registerInfo, CancellationToken token) { try { var nodeName = registerInfo.NodeName.ToLowerInvariant(); this.logger.LogInformation("RegisterRequested, NodeName {0}, Distro {1} ", nodeName, registerInfo.DistroInfo); var nodeTable = this.utilities.GetNodesTable(); var jsonTableEntity = new JsonTableEntity(this.utilities.NodesPartitionKey, this.utilities.GetRegistrationKey(nodeName), registerInfo); var result = await nodeTable.ExecuteAsync(TableOperation.InsertOrReplace(jsonTableEntity), null, null, token); using (HttpResponseMessage r = new HttpResponseMessage((HttpStatusCode)result.HttpStatusCode)) { r.EnsureSuccessStatusCode(); } // 5 minutes return(this.utilities.Option.RegistrationIntervalSeconds * 1000); } catch (Exception ex) { this.logger.LogError(ex, "RegisterRequested. NodeName {0}, Distro {1}", registerInfo.NodeName, registerInfo.DistroInfo); } return(this.utilities.Option.RetryOnFailureSeconds * 1000); }
public Task CreateParticipentAsync(Guid contestId, CreateParticipentModel participent) { var participentEntity = new JsonTableEntity <ParticipentModel>(contestId.ToString(), participent.UserName, new ParticipentModel(contestId, participent.UserName, participent.PushInfo)); var table = this.StorageAccessService.GetTableReference(TableName); var operation = TableOperation.Insert(participentEntity); return(table.ExecuteAsync(operation)); }
public async Task <Guid> CreateContestAsync(BeerContestModel contest) { Guid contestId = Guid.NewGuid(); contest.Id = contestId; var entity = new JsonTableEntity <BeerContestModel>(contestId.ToString(), contestId.ToString(), contest); var table = StorageAccessService.GetTableReference(TableName); TableOperation operation = TableOperation.Insert(entity); await table.ExecuteAsync(operation); return(contestId); }
public async T.Task RegisterNodeAsync(ComputeClusterRegistrationInformation info, CancellationToken token) { var nodeName = info.NodeName.ToLowerInvariant(); this.Logger.Information("RegisterRequested, NodeName {0}, Distro {1} ", nodeName, info.DistroInfo); var nodeTable = this.Utilities.GetNodesTable(); var jsonTableEntity = new JsonTableEntity(this.Utilities.NodesPartitionKey, this.Utilities.GetRegistrationKey(nodeName), info); var result = await nodeTable.ExecuteAsync(TableOperation.InsertOrReplace(jsonTableEntity), null, null, token); using (HttpResponseMessage r = new HttpResponseMessage((HttpStatusCode)result.HttpStatusCode)) { r.EnsureSuccessStatusCode(); } }
public async T.Task DoWorkAsync(CancellationToken token) { long currentMinute = 0; while (!token.IsCancellationRequested) { try { await T.Task.Delay(TimeSpan.FromSeconds(this.workerOptions.MetricsIntervalSeconds), token); var nodeName = this.ServerOptions.HostName; // TODO: different frequency IList <(string, string)> metricScripts = await this.GetMetricScriptsAsync(token); string toErrorJson(string e) => JsonConvert.SerializeObject(new Dictionary <string, string>() { { "Error", e } }, Formatting.Indented); var results = await T.Task.WhenAll(metricScripts.Select(async((string, string)s) => { try { this.Logger.Debug("Collect metrics for {0}", s.Item1); var scriptOutput = await PythonExecutor.ExecuteScriptAsync(s.Item2, null, token); return(s.Item1, scriptOutput.IsError ? toErrorJson(scriptOutput.ErrorMessage) : scriptOutput.Output); } catch (Exception ex) { return(s.Item1, toErrorJson(ex.ToString())); } })); DynamicTableEntity entity = new DynamicTableEntity( this.Utilities.MetricsValuesPartitionKey, nodeName, "*", results.ToDictionary( r => r.Item1, r => new EntityProperty(r.Item2))); var result = await metricsTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), null, null, token); if (!result.IsSuccessfulStatusCode()) { continue; } var nodesPartitionKey = this.Utilities.GetNodePartitionKey(nodeName); var time = DateTimeOffset.UtcNow; var minuteHistoryKey = this.Utilities.GetMinuteHistoryKey(); result = await this.nodesTable.ExecuteAsync(TableOperation.Retrieve <JsonTableEntity>(nodesPartitionKey, minuteHistoryKey), null, null, token); var history = result.Result is JsonTableEntity historyEntity?historyEntity.GetObject <MetricHistory>() : new MetricHistory(TimeSpan.FromSeconds(10)); var currentMetrics = results.Select(r => new MetricItem() { Category = r.Item1, InstanceValues = JsonConvert.DeserializeObject <Dictionary <string, double?> >(r.Item2) }).ToList(); history.RangeSeconds = 10; history.Put(time, currentMetrics); result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(new JsonTableEntity(nodesPartitionKey, minuteHistoryKey, history)), null, null, token); if (!result.IsSuccessfulStatusCode()) { continue; } var minute = time.UtcTicks / TimeSpan.TicksPerMinute; if (minute > currentMinute) { currentMinute = minute; // persist minute data var currentMetricsEntity = new JsonTableEntity(this.Utilities.GetNodePartitionKey(nodeName), this.Utilities.GetMinuteHistoryKey(currentMinute), currentMetrics); result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(currentMetricsEntity), null, null, token); if (!result.IsSuccessfulStatusCode()) { continue; } } }
public override async Task <bool> DoWorkAsync(TaskItem taskItem, CancellationToken token) { var job = taskItem.GetMessage <InternalJob>(); var nodeName = this.Configuration.GetValue <string>(Constants.HpcHostNameEnv); using (this.logger.BeginScope("Do work for InternalJob {0} on node {1}", job.Id, nodeName)) { // TODO: make sure invisible. logger.LogInformation("Executing job {0}", job.Id); var tasks = Enumerable.Range(0, job.CommandLines.Length).Select(async taskId => { var cmd = job.CommandLines[taskId]; logger.LogInformation("Executing command {0}, job {1}", cmd, job.Id); var taskKey = this.utilities.GetTaskKey(job.Id, taskId, job.RequeueCount); var resultKey = this.utilities.GetJobResultKey(nodeName, taskKey); var taskResultBlob = await this.utilities.CreateOrReplaceTaskOutputBlobAsync(job.Id, resultKey, token); using (var monitor = this.Monitor.StartMonitorTask(taskKey, async(output, cancellationToken) => { try { await taskResultBlob.AppendTextAsync(output, Encoding.UTF8, null, null, null, cancellationToken); } catch (Exception ex) { this.logger.LogError(ex, "Error happened when append to blob {0}", taskResultBlob.Name); } })) { this.logger.LogInformation("Call startjobandtask for job {0}, task {1}", job.Id, taskKey); var jobPartitionName = this.utilities.GetJobPartitionKey($"{job.Type}", job.Id); var nodePartitionName = this.utilities.GetNodePartitionKey(nodeName); var taskResultArgs = new ComputeNodeTaskCompletionEventArgs(nodeName, job.Id, null) { State = TaskState.Dispatching }; var taskResultEntity = new JsonTableEntity(jobPartitionName, resultKey, taskResultArgs); var result = await jobsTable.ExecuteAsync(TableOperation.InsertOrReplace(taskResultEntity), null, null, token); this.logger.LogInformation("Saved task result {0} to jobs table, status code {1}", resultKey, result.HttpStatusCode); if (!result.IsSuccessfulStatusCode()) { return(false); } var nodeResultEntity = new JsonTableEntity(nodePartitionName, resultKey, taskResultArgs); result = await nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(nodeResultEntity), null, null, token); this.logger.LogInformation("Saved task result {0} to nodes table, status code {1}", resultKey, result.HttpStatusCode); if (!result.IsSuccessfulStatusCode()) { return(false); } await this.communicator.StartJobAndTaskAsync( nodeName, new StartJobAndTaskArg(new int[0], job.Id, taskId), "", "", new ProcessStartInfo(cmd, "", "", $"{this.communicator.Options.AgentUriBase}/output/{taskKey}", "", new System.Collections.Hashtable(), new long[0], job.RequeueCount), token); taskResultArgs = new ComputeNodeTaskCompletionEventArgs(nodeName, job.Id, null) { State = TaskState.Running }; taskResultEntity = new JsonTableEntity(jobPartitionName, resultKey, taskResultArgs); result = await jobsTable.ExecuteAsync(TableOperation.InsertOrReplace(taskResultEntity), null, null, token); this.logger.LogInformation("Saved task result {0} to jobs table, status code {1}", resultKey, result.HttpStatusCode); if (!result.IsSuccessfulStatusCode()) { return(false); } nodeResultEntity = new JsonTableEntity(nodePartitionName, resultKey, taskResultArgs); result = await nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(nodeResultEntity), null, null, token); this.logger.LogInformation("Saved task result {0} to nodes table, status code {1}", resultKey, result.HttpStatusCode); if (!result.IsSuccessfulStatusCode()) { return(false); } this.logger.LogInformation("Wait for response for job {0}, task {1}", job.Id, taskKey); taskResultArgs = await monitor.Execution; this.logger.LogInformation("Saving result for job {0}, task {1}", job.Id, taskKey); taskResultArgs.State = TaskState.Finished; taskResultEntity = new JsonTableEntity(jobPartitionName, resultKey, taskResultArgs); result = await jobsTable.ExecuteAsync(TableOperation.InsertOrReplace(taskResultEntity), null, null, token); this.logger.LogInformation("Saved task result {0} to jobs table, status code {1}", resultKey, result.HttpStatusCode); if (!result.IsSuccessfulStatusCode()) { return(false); } nodeResultEntity = new JsonTableEntity(nodePartitionName, resultKey, taskResultArgs); result = await nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(nodeResultEntity), null, null, token); this.logger.LogInformation("Saved task result {0} to nodes table, status code {1}", resultKey, result.HttpStatusCode); if (!result.IsSuccessfulStatusCode()) { return(false); } return(true); } }); var results = await Task.WhenAll <bool>(tasks); return(results.All(r => r)); } }
public override async Task <bool> DoWorkAsync(TaskItem taskItem, CancellationToken token) { long currentMinute = 0; while (true) { await Task.Delay(this.config.GetValue <int>("MetricInterval"), token); try { var nodeName = this.config.GetValue <string>(Constants.HpcHostNameEnv); using (this.logger.BeginScope("do metrics loop on {0}", nodeName)) { // TODO: different frequency IList <(string, string)> metricScripts = await this.GetMetricScriptsAsync(token); string toErrorJson(string e) => JsonConvert.SerializeObject(new Dictionary <string, string>() { { "Error", e } }, Formatting.Indented); var results = await Task.WhenAll(metricScripts.Select(async s => { try { this.logger.LogDebug("Collect metrics for {0}", s.Item1); var psi = new System.Diagnostics.ProcessStartInfo( @"python", $"-c \"{s.Item2.Replace("\"", "\\\"")}\"") { UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, RedirectStandardInput = true, }; using (var process = new Process() { StartInfo = psi, EnableRaisingEvents = true }) { try { process.Start(); var output = await process.StandardOutput.ReadToEndAsync(); var error = await process.StandardError.ReadToEndAsync(); return(s.Item1, string.IsNullOrEmpty(error) ? output : toErrorJson(error)); } finally { process.Kill(); } } } catch (Exception ex) { return(s.Item1, toErrorJson(ex.ToString())); } })); DynamicTableEntity entity = new DynamicTableEntity( this.utilities.MetricsValuesPartitionKey, nodeName, "*", results.ToDictionary( r => r.Item1, r => new EntityProperty(r.Item2))); var result = await metricsTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), null, null, token); if (!result.IsSuccessfulStatusCode()) { continue; } var nodesPartitionKey = this.utilities.GetNodePartitionKey(nodeName); var time = DateTimeOffset.UtcNow; var minuteHistoryKey = this.utilities.GetMinuteHistoryKey(); result = await this.nodesTable.ExecuteAsync(TableOperation.Retrieve <JsonTableEntity>(nodesPartitionKey, minuteHistoryKey), null, null, token); var history = result.Result is JsonTableEntity historyEntity?historyEntity.GetObject <MetricHistory>() : new MetricHistory(TimeSpan.FromSeconds(10)); var currentMetrics = results.Select(r => new MetricItem() { Category = r.Item1, InstanceValues = JsonConvert.DeserializeObject <Dictionary <string, double?> >(r.Item2) }).ToList(); history.Range = TimeSpan.FromSeconds(10); history.Put(time, currentMetrics); result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(new JsonTableEntity(nodesPartitionKey, minuteHistoryKey, history)), null, null, token); if (!result.IsSuccessfulStatusCode()) { continue; } var minute = time.UtcTicks / TimeSpan.TicksPerMinute; if (minute > currentMinute) { currentMinute = minute; // persist minute data var currentMetricsEntity = new JsonTableEntity(this.utilities.GetNodePartitionKey(nodeName), this.utilities.GetMinuteHistoryKey(currentMinute), currentMetrics); result = await this.nodesTable.ExecuteAsync(TableOperation.InsertOrReplace(currentMetricsEntity), null, null, token); if (!result.IsSuccessfulStatusCode()) { continue; } } } } catch (Exception ex) { this.logger.LogError(ex, "DoWorkAsync error."); } } }
public async Task UpdateContestEntityAsync(JsonTableEntity <BeerContestModel> entity) { var table = StorageAccessService.GetTableReference(TableName); TableOperation operation = TableOperation.Replace(entity); await table.ExecuteAsync(operation); }