public ExtendedLivyBatchResponse SubmitSparkBatchJob(string workspaceName, string sparkPoolName, ExtendedLivyBatchRequest livyRequest, bool waitForCompletion) { var batch = _synapseClient.SparkBatch.Create(workspaceName, sparkPoolName, livyRequest, detailed: true); if (!waitForCompletion) { return(batch); } return(PollSparkBatchJobSubmission(workspaceName, sparkPoolName, batch)); }
/// <summary> /// Create new spark batch job. /// </summary> /// <param name='workspaceName'> /// The name of the workspace to execute operations on. /// </param> /// <param name='sparkPoolName'> /// Name of the spark pool. "ondemand" targets the ondemand pool. /// </param> /// <param name='livyRequest'> /// Livy compatible batch job request payload. /// </param> /// <param name='detailed'> /// Optional query param specifying whether detailed response is returned /// beyond plain livy. /// </param> /// <param name='customHeaders'> /// Headers that will be added to request. /// </param> /// <param name='cancellationToken'> /// The cancellation token. /// </param> /// <exception cref="CloudException"> /// Thrown when the operation returned an invalid status code /// </exception> /// <exception cref="SerializationException"> /// Thrown when unable to deserialize the response /// </exception> /// <exception cref="ValidationException"> /// Thrown when a required parameter is null /// </exception> /// <exception cref="System.ArgumentNullException"> /// Thrown when a required parameter is null /// </exception> /// <return> /// A response object containing the response body and response headers. /// </return> public async Task <AzureOperationResponse <ExtendedLivyBatchResponse> > CreateWithHttpMessagesAsync(string workspaceName, string sparkPoolName, ExtendedLivyBatchRequest livyRequest, bool?detailed = default(bool?), Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken)) { if (workspaceName == null) { throw new ValidationException(ValidationRules.CannotBeNull, "workspaceName"); } if (Client.SynapseDnsSuffix == null) { throw new ValidationException(ValidationRules.CannotBeNull, "this.Client.SynapseDnsSuffix"); } if (Client.LivyApiVersion == null) { throw new ValidationException(ValidationRules.CannotBeNull, "this.Client.LivyApiVersion"); } if (sparkPoolName == null) { throw new ValidationException(ValidationRules.CannotBeNull, "sparkPoolName"); } if (livyRequest == null) { throw new ValidationException(ValidationRules.CannotBeNull, "livyRequest"); } // Tracing bool _shouldTrace = ServiceClientTracing.IsEnabled; string _invocationId = null; if (_shouldTrace) { _invocationId = ServiceClientTracing.NextInvocationId.ToString(); Dictionary <string, object> tracingParameters = new Dictionary <string, object>(); tracingParameters.Add("workspaceName", workspaceName); tracingParameters.Add("sparkPoolName", sparkPoolName); tracingParameters.Add("detailed", detailed); tracingParameters.Add("livyRequest", livyRequest); tracingParameters.Add("cancellationToken", cancellationToken); ServiceClientTracing.Enter(_invocationId, this, "Create", tracingParameters); } // Construct URL var _baseUrl = Client.BaseUri; var _url = _baseUrl + (_baseUrl.EndsWith("/") ? "" : "/") + "livyApi/versions/{livyApiVersion}/sparkPools/{sparkPoolName}/batches"; _url = _url.Replace("{workspaceName}", workspaceName); _url = _url.Replace("{SynapseDnsSuffix}", Client.SynapseDnsSuffix); _url = _url.Replace("{livyApiVersion}", Client.LivyApiVersion); _url = _url.Replace("{sparkPoolName}", System.Uri.EscapeDataString(sparkPoolName)); List <string> _queryParameters = new List <string>(); if (detailed != null) { _queryParameters.Add(string.Format("detailed={0}", System.Uri.EscapeDataString(Rest.Serialization.SafeJsonConvert.SerializeObject(detailed, Client.SerializationSettings).Trim('"')))); } if (_queryParameters.Count > 0) { _url += (_url.Contains("?") ? "&" : "?") + string.Join("&", _queryParameters); } // Create HTTP transport objects var _httpRequest = new HttpRequestMessage(); HttpResponseMessage _httpResponse = null; _httpRequest.Method = new HttpMethod("POST"); _httpRequest.RequestUri = new System.Uri(_url); // Set Headers if (Client.GenerateClientRequestId != null && Client.GenerateClientRequestId.Value) { _httpRequest.Headers.TryAddWithoutValidation("x-ms-client-request-id", System.Guid.NewGuid().ToString()); } if (Client.AcceptLanguage != null) { if (_httpRequest.Headers.Contains("accept-language")) { _httpRequest.Headers.Remove("accept-language"); } _httpRequest.Headers.TryAddWithoutValidation("accept-language", Client.AcceptLanguage); } if (customHeaders != null) { foreach (var _header in customHeaders) { if (_httpRequest.Headers.Contains(_header.Key)) { _httpRequest.Headers.Remove(_header.Key); } _httpRequest.Headers.TryAddWithoutValidation(_header.Key, _header.Value); } } // Serialize Request string _requestContent = null; if (livyRequest != null) { _requestContent = Rest.Serialization.SafeJsonConvert.SerializeObject(livyRequest, Client.SerializationSettings); _httpRequest.Content = new StringContent(_requestContent, System.Text.Encoding.UTF8); _httpRequest.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json; charset=utf-8"); } // Set Credentials if (Client.Credentials != null) { cancellationToken.ThrowIfCancellationRequested(); await Client.Credentials.ProcessHttpRequestAsync(_httpRequest, cancellationToken).ConfigureAwait(false); } // Send Request if (_shouldTrace) { ServiceClientTracing.SendRequest(_invocationId, _httpRequest); } cancellationToken.ThrowIfCancellationRequested(); _httpResponse = await Client.HttpClient.SendAsync(_httpRequest, cancellationToken).ConfigureAwait(false); if (_shouldTrace) { ServiceClientTracing.ReceiveResponse(_invocationId, _httpResponse); } HttpStatusCode _statusCode = _httpResponse.StatusCode; cancellationToken.ThrowIfCancellationRequested(); string _responseContent = null; if ((int)_statusCode != 200) { var ex = new CloudException(string.Format("Operation returned an invalid status code '{0}'", _statusCode)); try { _responseContent = await _httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false); CloudError _errorBody = Rest.Serialization.SafeJsonConvert.DeserializeObject <CloudError>(_responseContent, Client.DeserializationSettings); if (_errorBody != null) { ex = new CloudException(_errorBody.Message); ex.Body = _errorBody; } } catch (JsonException) { // Ignore the exception } ex.Request = new HttpRequestMessageWrapper(_httpRequest, _requestContent); ex.Response = new HttpResponseMessageWrapper(_httpResponse, _responseContent); if (_httpResponse.Headers.Contains("x-ms-request-id")) { ex.RequestId = _httpResponse.Headers.GetValues("x-ms-request-id").FirstOrDefault(); } if (_shouldTrace) { ServiceClientTracing.Error(_invocationId, ex); } _httpRequest.Dispose(); if (_httpResponse != null) { _httpResponse.Dispose(); } throw ex; } // Create Result var _result = new AzureOperationResponse <ExtendedLivyBatchResponse>(); _result.Request = _httpRequest; _result.Response = _httpResponse; if (_httpResponse.Headers.Contains("x-ms-request-id")) { _result.RequestId = _httpResponse.Headers.GetValues("x-ms-request-id").FirstOrDefault(); } // Deserialize Response if ((int)_statusCode == 200) { _responseContent = await _httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false); try { _result.Body = Rest.Serialization.SafeJsonConvert.DeserializeObject <ExtendedLivyBatchResponse>(_responseContent, Client.DeserializationSettings); } catch (JsonException ex) { _httpRequest.Dispose(); if (_httpResponse != null) { _httpResponse.Dispose(); } throw new SerializationException("Unable to deserialize the response.", _responseContent, ex); } } if (_shouldTrace) { ServiceClientTracing.Exit(_invocationId, _result); } return(_result); }
public override void ExecuteCmdlet() { this.Language = LanguageType.Parse(this.Language); if (string.IsNullOrEmpty(this.MainClassName)) { if (LanguageType.SparkDotNet == this.Language || LanguageType.Spark == this.Language) { throw new SynapseException(Resources.MissingMainClassName); } } if (this.IsParameterBound(c => c.SparkPoolObject)) { var resourceIdentifier = new ResourceIdentifier(this.SparkPoolObject.Id); this.WorkspaceName = resourceIdentifier.ParentResource; this.WorkspaceName = this.WorkspaceName.Substring(this.WorkspaceName.LastIndexOf('/') + 1); this.SparkPoolName = resourceIdentifier.ResourceName; } this.MainDefinitionFile = Utils.NormalizeUrl(this.MainDefinitionFile); if (this.CommandLineArgument != null) { for (int i = 0; i < this.CommandLineArgument.Length; i++) { this.CommandLineArgument[i] = Utils.NormalizeUrl(this.CommandLineArgument[i]); } } if (this.ReferenceFile != null) { for (int i = 0; i < this.ReferenceFile.Length; i++) { this.ReferenceFile[i] = Utils.NormalizeUrl(this.ReferenceFile[i]); } } Utils.CategorizedFiles(this.ReferenceFile, out IList <string> jars, out IList <string> files); bool isSparkDotNet = this.Language == LanguageType.SparkDotNet; var batchRequest = new ExtendedLivyBatchRequest { Name = this.Name, File = isSparkDotNet ? SynapseConstants.SparkDotNetJarFile : this.MainDefinitionFile, ClassName = isSparkDotNet ? SynapseConstants.SparkDotNetClassName : (this.Language == LanguageType.PySpark ? null : this.MainClassName), Args = isSparkDotNet ? new List <string> { this.MainDefinitionFile, this.MainClassName } .Concat(this.CommandLineArgument ?? new string[0]).ToArray() : this.CommandLineArgument, Jars = jars, Files = files, Archives = isSparkDotNet ? new List <string> { $"{this.MainDefinitionFile}#{SynapseConstants.SparkDotNetUdfsFolderName}" } : null, Conf = this.Configuration?.ToDictionary(), ExecutorMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g", ExecutorCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores, DriverMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g", DriverCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores, NumExecutors = this.ExecutorCount }; // Ensure the relative path of UDFs is add to "--conf". if (isSparkDotNet) { batchRequest.Conf = batchRequest.Conf ?? new Dictionary <string, string>(); string udfsRelativePath = "./" + SynapseConstants.SparkDotNetUdfsFolderName; batchRequest.Conf.TryGetValue(SynapseConstants.SparkDotNetAssemblySearchPathsKey, out string pathValue); var paths = pathValue?.Split(',').Select(path => path.Trim()).Where(path => !string.IsNullOrEmpty(path)).ToList() ?? new List <string>(); if (!paths.Contains(udfsRelativePath)) { paths.Add(udfsRelativePath); } batchRequest.Conf[SynapseConstants.SparkDotNetAssemblySearchPathsKey] = string.Join(",", paths); } var jobInformation = SynapseAnalyticsClient.SubmitSparkBatchJob(this.WorkspaceName, this.SparkPoolName, batchRequest, waitForCompletion: false); WriteObject(new PSSynapseSparkJob(jobInformation)); }
public override void ExecuteCmdlet() { this.Language = LanguageType.Parse(this.Language); if (string.IsNullOrEmpty(this.MainClassName)) { if (LanguageType.SparkDotNet == this.Language || LanguageType.Spark == this.Language) { throw new SynapseException(Resources.MissingMainClassName); } } if (this.IsParameterBound(c => c.SparkPoolObject)) { var resourceIdentifier = new ResourceIdentifier(this.SparkPoolObject.Id); this.WorkspaceName = resourceIdentifier.ParentResource; this.WorkspaceName = this.WorkspaceName.Substring(this.WorkspaceName.LastIndexOf('/') + 1); this.SparkPoolName = resourceIdentifier.ResourceName; } this.MainDefinitionFile = Utils.NormalizeUrl(this.MainDefinitionFile); if (this.CommandLineArguments != null) { for (int i = 0; i < this.CommandLineArguments.Length; i++) { this.CommandLineArguments[i] = Utils.NormalizeUrl(this.CommandLineArguments[i]); } } if (this.ReferenceFiles != null) { for (int i = 0; i < this.ReferenceFiles.Length; i++) { this.ReferenceFiles[i] = Utils.NormalizeUrl(this.ReferenceFiles[i]); } } Utils.CategorizedFiles(this.ReferenceFiles, out IList <string> jars, out IList <string> files); bool isSparkDotNet = this.Language == LanguageType.SparkDotNet; var batchRequest = new ExtendedLivyBatchRequest { Name = this.Name, File = isSparkDotNet ? SynapseConstants.SparkDotNetJarFile : this.MainDefinitionFile, ClassName = isSparkDotNet ? SynapseConstants.SparkDotNetClassName : (this.Language == LanguageType.PySpark ? null : this.MainClassName), Args = isSparkDotNet ? new List <string> { this.MainDefinitionFile, this.MainClassName } .Concat(this.CommandLineArguments ?? new string[0]).ToArray() : this.CommandLineArguments, Jars = jars, Files = files, Archives = isSparkDotNet ? new List <string> { $"{this.MainDefinitionFile}#{SynapseConstants.SparkDotNetUdfsFolderName}" } : null, Conf = this.Configuration?.ToDictionary(), ExecutorMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g", ExecutorCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores, DriverMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g", DriverCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores, NumExecutors = this.ExecutorCount }; var jobInformation = SynapseAnalyticsClient.SubmitSparkBatchJob(this.WorkspaceName, this.SparkPoolName, batchRequest, waitForCompletion: false); WriteObject(new PSSynapseSparkJob(jobInformation)); }
/// <summary> /// Create new spark batch job. /// </summary> /// <param name='operations'> /// The operations group for this extension method. /// </param> /// <param name='workspaceName'> /// The name of the workspace to execute operations on. /// </param> /// <param name='sparkPoolName'> /// Name of the spark pool. "ondemand" targets the ondemand pool. /// </param> /// <param name='livyRequest'> /// Livy compatible batch job request payload. /// </param> /// <param name='detailed'> /// Optional query param specifying whether detailed response is returned /// beyond plain livy. /// </param> /// <param name='cancellationToken'> /// The cancellation token. /// </param> public static async Task <ExtendedLivyBatchResponse> CreateAsync(this ISparkBatchOperations operations, string workspaceName, string sparkPoolName, ExtendedLivyBatchRequest livyRequest, bool?detailed = default(bool?), CancellationToken cancellationToken = default(CancellationToken)) { using (var _result = await operations.CreateWithHttpMessagesAsync(workspaceName, sparkPoolName, livyRequest, detailed, null, cancellationToken).ConfigureAwait(false)) { return(_result.Body); } }
/// <summary> /// Create new spark batch job. /// </summary> /// <param name='operations'> /// The operations group for this extension method. /// </param> /// <param name='workspaceName'> /// The name of the workspace to execute operations on. /// </param> /// <param name='sparkPoolName'> /// Name of the spark pool. "ondemand" targets the ondemand pool. /// </param> /// <param name='livyRequest'> /// Livy compatible batch job request payload. /// </param> /// <param name='detailed'> /// Optional query param specifying whether detailed response is returned /// beyond plain livy. /// </param> public static ExtendedLivyBatchResponse Create(this ISparkBatchOperations operations, string workspaceName, string sparkPoolName, ExtendedLivyBatchRequest livyRequest, bool?detailed = default(bool?)) { return(operations.CreateAsync(workspaceName, sparkPoolName, livyRequest, detailed).GetAwaiter().GetResult()); }