public override async Task <ClusterJob> SubmitJobAsync(IClusterConnection connection, IEnumerable <KeyValuePair <string, object> > properties, CancellationToken cancellationToken = default) { var connectionInformation = connection.As <SparkClusterConnection>() ?? throw new InvalidCastException($"Current connection type {connection.GetType().Name} cannot be converted into {typeof(SparkClusterConnection).Name}"); dynamic payload = new ExpandoObject(); foreach (var kvp in properties) { ((IDictionary <string, object>)payload)[kvp.Key] = kvp.Value; } if (connectionInformation.Properties?.Count() > 0) { var conf = new Dictionary <string, object>(connectionInformation.Properties); ((IDictionary <string, object>)payload)["conf"] = conf; } var payloadJson = JsonConvert.SerializeObject(payload); var postBatchesUrl = BuildEndpointUrl(connectionInformation.BaseUrl, "batches"); using var responseMessage = await _httpClient.PostAsync(postBatchesUrl, new StringContent (payloadJson, Encoding.UTF8, "application/json"), cancellationToken); if (responseMessage.StatusCode == HttpStatusCode.Created) { var responseJson = await responseMessage.Content.ReadAsStringAsync(); var responseObj = JObject.Parse(responseJson); var sparkBatchId = responseObj["id"]?.Value <int>(); var sparkBatchName = responseObj["name"]?.Value <string>(); return(new ClusterJob { ConnectionId = connectionInformation.Id, LocalJobId = sparkBatchId.ToString(), Name = sparkBatchName, State = ClusterJobState.Created }); } else { var errorMessage = await responseMessage.Content.ReadAsStringAsync(); throw new ClusterJobException($"Failed to create the job. Details: {errorMessage}"); } }
public override async Task <ClusterJob> GetJobAsync(IClusterConnection connection, string localJobId, CancellationToken cancellationToken = default) { var connectionInformation = connection.As <SparkClusterConnection>() ?? throw new InvalidCastException($"Current connection type {connection.GetType().Name} cannot be converted into {typeof(SparkClusterConnection).Name}"); var retrieveBatchUrl = BuildEndpointUrl(connectionInformation.BaseUrl, $"batches/{localJobId}"); HttpResponseMessage responseMessage; using (responseMessage = await _httpClient.GetAsync(retrieveBatchUrl, cancellationToken)) { try { responseMessage.EnsureSuccessStatusCode(); var responseJson = await responseMessage.Content.ReadAsStringAsync(); var responseObj = JObject.Parse(responseJson); var jobName = responseObj["name"]?.Value <string>(); var jobState = ConvertToJobState(responseObj["state"]?.Value <string>()); var retrieveBatchLogUrl = BuildEndpointUrl(connectionInformation.BaseUrl, $"batches/{localJobId}/log"); responseMessage = await _httpClient.GetAsync(retrieveBatchLogUrl); responseMessage.EnsureSuccessStatusCode(); var logResponseJson = await responseMessage.Content.ReadAsStringAsync(); var logResponseObj = JObject.Parse(logResponseJson); var jobLogs = logResponseObj["log"]?.ToObject <string[]>(); return(new ClusterJob(connection.Id, localJobId) { Name = jobName, State = jobState, Logs = jobLogs.ToList() }); } catch (HttpRequestException ex) when(responseMessage?.StatusCode == HttpStatusCode.NotFound) { throw new ClusterJobException($"The job {localJobId} doesn't exist on cluster {connection.Id}.", ex); } catch (Exception ex) { throw new ClusterJobException(ex.Message, ex); } } }
public override async Task <ClusterState> GetStateAsync(IClusterConnection connection, CancellationToken cancellationToken = default) { try { var connectionInformation = connection.As <SparkClusterConnection>() ?? throw new ArgumentException($"Current connection type {connection.GetType().Name} cannot be converted into {typeof(SparkClusterConnection).Name}"); using var responseMessage = await _httpClient.GetAsync(connectionInformation.BaseUrl, cancellationToken); responseMessage.EnsureSuccessStatusCode(); return(ClusterState.Online); } catch { return(ClusterState.Offline); } }