public override async Task <ClusterJob> SubmitJobAsync(IClusterConnection connection, IEnumerable <KeyValuePair <string, object> > properties, CancellationToken cancellationToken = default)
{
var connectionInformation = connection.As <SparkClusterConnection>()
??
throw new InvalidCastException($"Current connection type {connection.GetType().Name} cannot be converted into {typeof(SparkClusterConnection).Name}");
dynamic payload = new ExpandoObject();
foreach (var kvp in properties)
{
((IDictionary <string, object>)payload)[kvp.Key] = kvp.Value;
}
if (connectionInformation.Properties?.Count() > 0)
{
var conf = new Dictionary <string, object>(connectionInformation.Properties);
((IDictionary <string, object>)payload)["conf"] = conf;
}
var payloadJson = JsonConvert.SerializeObject(payload);
var postBatchesUrl = BuildEndpointUrl(connectionInformation.BaseUrl, "batches");
using var responseMessage = await _httpClient.PostAsync(postBatchesUrl,
new StringContent (payloadJson, Encoding.UTF8, "application/json"),
cancellationToken);
if (responseMessage.StatusCode == HttpStatusCode.Created)
{
var responseJson = await responseMessage.Content.ReadAsStringAsync();
var responseObj = JObject.Parse(responseJson);
var sparkBatchId = responseObj["id"]?.Value <int>();
var sparkBatchName = responseObj["name"]?.Value <string>();
return(new ClusterJob
{
ConnectionId = connectionInformation.Id,
LocalJobId = sparkBatchId.ToString(),
Name = sparkBatchName,
State = ClusterJobState.Created
});
}
else
{
var errorMessage = await responseMessage.Content.ReadAsStringAsync();
throw new ClusterJobException($"Failed to create the job. Details: {errorMessage}");
}
}
public override async Task <ClusterJob> GetJobAsync(IClusterConnection connection, string localJobId, CancellationToken cancellationToken = default)
{
var connectionInformation = connection.As <SparkClusterConnection>()
??
throw new InvalidCastException($"Current connection type {connection.GetType().Name} cannot be converted into {typeof(SparkClusterConnection).Name}");
var retrieveBatchUrl = BuildEndpointUrl(connectionInformation.BaseUrl, $"batches/{localJobId}");
HttpResponseMessage responseMessage;
using (responseMessage = await _httpClient.GetAsync(retrieveBatchUrl, cancellationToken))
{
try
{
responseMessage.EnsureSuccessStatusCode();
var responseJson = await responseMessage.Content.ReadAsStringAsync();
var responseObj = JObject.Parse(responseJson);
var jobName = responseObj["name"]?.Value <string>();
var jobState = ConvertToJobState(responseObj["state"]?.Value <string>());
var retrieveBatchLogUrl = BuildEndpointUrl(connectionInformation.BaseUrl, $"batches/{localJobId}/log");
responseMessage = await _httpClient.GetAsync(retrieveBatchLogUrl);
responseMessage.EnsureSuccessStatusCode();
var logResponseJson = await responseMessage.Content.ReadAsStringAsync();
var logResponseObj = JObject.Parse(logResponseJson);
var jobLogs = logResponseObj["log"]?.ToObject <string[]>();
return(new ClusterJob(connection.Id, localJobId)
{
Name = jobName,
State = jobState,
Logs = jobLogs.ToList()
});
}
catch (HttpRequestException ex) when(responseMessage?.StatusCode == HttpStatusCode.NotFound)
{
throw new ClusterJobException($"The job {localJobId} doesn't exist on cluster {connection.Id}.", ex);
}
catch (Exception ex)
{
throw new ClusterJobException(ex.Message, ex);
}
}
}
public override async Task <ClusterState> GetStateAsync(IClusterConnection connection, CancellationToken cancellationToken = default)
{
try
{
var connectionInformation = connection.As <SparkClusterConnection>()
??
throw new ArgumentException($"Current connection type {connection.GetType().Name} cannot be converted into {typeof(SparkClusterConnection).Name}");
using var responseMessage = await _httpClient.GetAsync(connectionInformation.BaseUrl, cancellationToken);
responseMessage.EnsureSuccessStatusCode();
return(ClusterState.Online);
}
catch
{
return(ClusterState.Offline);
}
}
