private void SendRemote(CommandJob job, CommandJobTask subtask, StitchSummary stitch)
{
var request = CreateRemoteRequest(stitch);
request.Target = stitch.Id;
_sender.SendCommandRequest(stitch.NetworkNodeId, request, job, subtask);
}
public void Status_Test()
{
var target = new CommandJob();
target.Status.Should().Be(JobStatusType.Success);
target.Tasks.Add(new CommandJobTask
{
Status = JobStatusType.Success
});
target.Status.Should().Be(JobStatusType.Success);
target.Tasks.Add(new CommandJobTask
{
Status = JobStatusType.Started
});
target.Status.Should().Be(JobStatusType.Started);
target.Tasks.Add(new CommandJobTask
{
Status = JobStatusType.Failure
});
target.Status.Should().Be(JobStatusType.Failure);
}
protected override void SendLocal(CommandJob job, CommandJobTask subtask, StitchSummary stitch)
{
// TODO: Publish status async here and rely on the job to communicate status of the
// request. We will need a new mechanism for this and new message types
bool ok = _stitches.StopInstance(stitch.Id);
subtask.Status = ok ? JobStatusType.Success : JobStatusType.Failure;
}
public void Save(CommandJob job)
{
_data.Save(job, true);
if (job.IsComplete)
{
OnJobComplete(job.Id, job);
}
}
public CommandJob CreateJob(string name = null)
{
var job = new CommandJob
{
Id = Guid.NewGuid().ToString()
};
job.Name = name ?? job.Id;
return(job);
}
public void CreateSubtask_Test()
{
var target = new CommandJob();
var result = target.CreateSubtask(CommandType.Ping, "ABC123", "DEF456");
result.Id.Should().NotBeNullOrEmpty();
result.Status.Should().Be(JobStatusType.Started);
result.HandlingNodeId.Should().Be("DEF456");
result.Target.Should().Be("ABC123");
target.Tasks.Count.Should().Be(1);
}
private void OnJobComplete(string jobId, CommandJob job)
{
_log.LogDebug("Job Id={0} is complete: {1}", jobId, job.Status);
var completeEvent = new JobCompleteEvent
{
JobId = job.Id,
Status = job.Status
};
string channel = completeEvent.Status == JobStatusType.Success ? JobCompleteEvent.ChannelSuccess : JobCompleteEvent.ChannelFailure;
_messageBus.Publish(channel, completeEvent);
}
public void MarkTaskComplete_Test()
{
var target = new CommandJob();
var task = target.CreateSubtask(CommandType.Ping, "ABC", "123");
task.Status.Should().Be(JobStatusType.Started);
target.MarkTaskComplete("INVALID", true);
task.Status.Should().Be(JobStatusType.Started);
target.MarkTaskComplete(task.Id, true);
task.Status.Should().Be(JobStatusType.Success);
target.MarkTaskComplete(task.Id, false);
task.Status.Should().Be(JobStatusType.Failure);
}
static async Task Main(
string subscriptionId = "48bbc269-ce89-4f6f-9a12-c6f91fcb772d",
string resourceGroupName = "aml1p-rg",
string workspaceName = "aml1p-ml-wus2",
string clusterName = "cpucluster"
)
{
// https://github.com/Azure/azure-sdk-for-net/blob/main/doc/dev/mgmt_quickstart.md
var credential = new DefaultAzureCredential();
var armClient = new ArmClient(credential);
var subscription = armClient.GetSubscriptionResource(new ResourceIdentifier($"/subscriptions/{subscriptionId}"));
var resourceGroup = await subscription.GetResourceGroupAsync(resourceGroupName);
var workspace = await resourceGroup.Value.GetMachineLearningWorkspaceAsync(workspaceName);
var mlFlow = workspace.Value.Data.MlFlowTrackingUri;
Console.WriteLine($"MLFlow tracking URI: {mlFlow}");
var jobs = workspace.Value.GetMachineLearningJobs();
foreach (var job in jobs.Take(2))
{
var props = job.Data.Properties;
Console.WriteLine($"{props.DisplayName} ({props.Status}) == {props.Description}");
}
// https://github.com/Azure/azureml-samples-dotnet/blob/main/Program.cs
var commandJob = new CommandJob("printenv", "azureml://registries/CuratedRegistry/environments/AzureML-sklearn-1.0-ubuntu20.04-py38-cpu/versions/23")
{
ComputeId = $"{workspace.Value.Id}/computes/{clusterName}",
ExperimentName = "danmill-csharp"
};
var jobData = new MachineLearningJobData(commandJob);
var jobOperation = await jobs.CreateOrUpdateAsync(WaitUntil.Completed, Guid.NewGuid().ToString(), jobData);
var jobResource = jobOperation.Value;
Console.WriteLine($"Created job {jobResource.Data.Id}.");
}
protected abstract void SendLocal(CommandJob job, CommandJobTask subtask, StitchSummary stitch);
public void SendCommandRequest(string networkNodeId, CommandRequest request, CommandJob job, CommandJobTask task)
{
request.SetJobDetails(job.Id, task.Id);
var message = new ClusterMessageBuilder()
.FromNode()
.ToNode(networkNodeId)
.WithObjectPayload(request)
.Build();
Send(message);
}
