public async Task <LinkedServiceCreateOrUpdateResponse> GetCreateOrUpdateStatusAsync( string operationStatusLink, CancellationToken cancellationToken) { Ensure.IsNotNull(operationStatusLink, "operationStatusLink"); Core.Models.LinkedServiceCreateOrUpdateResponse internalResponse = await this.Client.InternalClient.LinkedServices.GetCreateOrUpdateStatusAsync( operationStatusLink, cancellationToken); var response = new LinkedServiceCreateOrUpdateResponse(internalResponse, this.Client); if (response.LinkedService != null && response.LinkedService.Properties != null && response.LinkedService.Properties.ProvisioningState != null) { response.Status = response.LinkedService.Properties.ProvisioningState.ToOperationStatus(); } return(response); }
/// <summary> /// Create or update a linked service. /// </summary> /// <param name='resourceGroupName'> /// Required. The resource group name of the linked service. /// </param> /// <param name='workspaceName'> /// Required. The name of the parent workspace that will contain the /// linked service /// </param> /// <param name='parameters'> /// Required. The parameters required to create or update a linked /// service. /// </param> /// <param name='cancellationToken'> /// Cancellation token. /// </param> /// <returns> /// The create or update linked service operation response. /// </returns> public async Task <LinkedServiceCreateOrUpdateResponse> CreateOrUpdateAsync(string resourceGroupName, string workspaceName, LinkedServiceCreateOrUpdateParameters parameters, CancellationToken cancellationToken) { // Validate if (resourceGroupName == null) { throw new ArgumentNullException("resourceGroupName"); } if (workspaceName == null) { throw new ArgumentNullException("workspaceName"); } if (parameters == null) { throw new ArgumentNullException("parameters"); } if (parameters.Name == null) { throw new ArgumentNullException("parameters.Name"); } if (parameters.Properties == null) { throw new ArgumentNullException("parameters.Properties"); } if (parameters.Properties.ResourceId == null) { throw new ArgumentNullException("parameters.Properties.ResourceId"); } // Tracing bool shouldTrace = TracingAdapter.IsEnabled; string invocationId = null; if (shouldTrace) { invocationId = TracingAdapter.NextInvocationId.ToString(); Dictionary <string, object> tracingParameters = new Dictionary <string, object>(); tracingParameters.Add("resourceGroupName", resourceGroupName); tracingParameters.Add("workspaceName", workspaceName); tracingParameters.Add("parameters", parameters); TracingAdapter.Enter(invocationId, this, "CreateOrUpdateAsync", tracingParameters); } // Construct URL string url = ""; url = url + "/subscriptions/"; if (this.Client.Credentials.SubscriptionId != null) { url = url + Uri.EscapeDataString(this.Client.Credentials.SubscriptionId); } url = url + "/resourcegroups/"; url = url + Uri.EscapeDataString(resourceGroupName); url = url + "/providers/Microsoft.OperationalInsights/workspaces/"; url = url + Uri.EscapeDataString(workspaceName); url = url + "/linkedServices/"; url = url + Uri.EscapeDataString(parameters.Name); List <string> queryParameters = new List <string>(); queryParameters.Add("api-version=2015-11-01-preview"); if (queryParameters.Count > 0) { url = url + "?" + string.Join("&", queryParameters); } string baseUrl = this.Client.BaseUri.AbsoluteUri; // Trim '/' character from the end of baseUrl and beginning of url. if (baseUrl[baseUrl.Length - 1] == '/') { baseUrl = baseUrl.Substring(0, baseUrl.Length - 1); } if (url[0] == '/') { url = url.Substring(1); } url = baseUrl + "/" + url; url = url.Replace(" ", "%20"); // Create HTTP transport objects HttpRequestMessage httpRequest = null; try { httpRequest = new HttpRequestMessage(); httpRequest.Method = HttpMethod.Put; httpRequest.RequestUri = new Uri(url); // Set Headers httpRequest.Headers.Add("x-ms-client-request-id", Guid.NewGuid().ToString()); // Set Credentials cancellationToken.ThrowIfCancellationRequested(); await this.Client.Credentials.ProcessHttpRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false); // Serialize Request string requestContent = null; JToken requestDoc = null; JObject linkedServiceCreateOrUpdateParametersValue = new JObject(); requestDoc = linkedServiceCreateOrUpdateParametersValue; linkedServiceCreateOrUpdateParametersValue["name"] = parameters.Name; JObject propertiesValue = new JObject(); linkedServiceCreateOrUpdateParametersValue["properties"] = propertiesValue; propertiesValue["resourceId"] = parameters.Properties.ResourceId; requestContent = requestDoc.ToString(Newtonsoft.Json.Formatting.Indented); httpRequest.Content = new StringContent(requestContent, Encoding.UTF8); httpRequest.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json"); // Send Request HttpResponseMessage httpResponse = null; try { if (shouldTrace) { TracingAdapter.SendRequest(invocationId, httpRequest); } cancellationToken.ThrowIfCancellationRequested(); httpResponse = await this.Client.HttpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false); if (shouldTrace) { TracingAdapter.ReceiveResponse(invocationId, httpResponse); } HttpStatusCode statusCode = httpResponse.StatusCode; if (statusCode != HttpStatusCode.OK && statusCode != HttpStatusCode.Created) { cancellationToken.ThrowIfCancellationRequested(); CloudException ex = CloudException.Create(httpRequest, requestContent, httpResponse, await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false)); if (shouldTrace) { TracingAdapter.Error(invocationId, ex); } throw ex; } // Create Result LinkedServiceCreateOrUpdateResponse result = null; // Deserialize Response if (statusCode == HttpStatusCode.OK || statusCode == HttpStatusCode.Created) { cancellationToken.ThrowIfCancellationRequested(); string responseContent = await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false); result = new LinkedServiceCreateOrUpdateResponse(); JToken responseDoc = null; if (string.IsNullOrEmpty(responseContent) == false) { responseDoc = JToken.Parse(responseContent); } if (responseDoc != null && responseDoc.Type != JTokenType.Null) { LinkedService linkedServiceInstance = new LinkedService(); result.LinkedService = linkedServiceInstance; JToken idValue = responseDoc["id"]; if (idValue != null && idValue.Type != JTokenType.Null) { string idInstance = ((string)idValue); linkedServiceInstance.Id = idInstance; } JToken nameValue = responseDoc["name"]; if (nameValue != null && nameValue.Type != JTokenType.Null) { string nameInstance = ((string)nameValue); linkedServiceInstance.Name = nameInstance; } JToken typeValue = responseDoc["type"]; if (typeValue != null && typeValue.Type != JTokenType.Null) { string typeInstance = ((string)typeValue); linkedServiceInstance.Type = typeInstance; } JToken propertiesValue2 = responseDoc["properties"]; if (propertiesValue2 != null && propertiesValue2.Type != JTokenType.Null) { LinkedServiceProperties propertiesInstance = new LinkedServiceProperties(); linkedServiceInstance.Properties = propertiesInstance; JToken resourceIdValue = propertiesValue2["resourceId"]; if (resourceIdValue != null && resourceIdValue.Type != JTokenType.Null) { string resourceIdInstance = ((string)resourceIdValue); propertiesInstance.ResourceId = resourceIdInstance; } } } } result.StatusCode = statusCode; if (httpResponse.Headers.Contains("x-ms-request-id")) { result.RequestId = httpResponse.Headers.GetValues("x-ms-request-id").FirstOrDefault(); } if (shouldTrace) { TracingAdapter.Exit(invocationId, result); } return(result); } finally { if (httpResponse != null) { httpResponse.Dispose(); } } } finally { if (httpRequest != null) { httpRequest.Dispose(); } } }
/// <summary> /// Deploys the specified ADF files and custom activity packages to Azure. /// </summary> /// <param name="filesToProcess">The files to process.</param> /// <param name="outputFolder">The output folder which contains the files and custom activity zips.</param> /// <param name="deployConfig">The deployment configuration information.</param> public async Task <bool> Deploy(List <string> filesToProcess, string outputFolder, DeployConfigInfo deployConfig = null) { bool result = true; logger.Write(string.Empty); logger.Write($"Getting all ADF resources to deploy to Azure from output folder '{outputFolder}'", "Black"); List <Task <AdfFileInfo> > allFilesTasks = filesToProcess.Select(async x => await adfFileHelper.GetFileInfo(x, deployConfig)).ToList(); List <AdfFileInfo> allFiles = new List <AdfFileInfo>(); foreach (var allFilesTask in allFilesTasks) { allFiles.Add(await allFilesTask); } List <AdfFileInfo> validFiles = allFiles.Where(x => x.IsValid).ToList(); if (!validFiles.Any()) { logger.Write($"No valid ADF files found in '{outputFolder}'", "Red"); return(false); } logger.Write($"{validFiles.Count} file{(validFiles.Count == 1 ? string.Empty : "s")} retreived"); logger.Write(string.Empty); logger.Write($"Begin deploying ADF resources to '{dataFactoryName}'", "Black"); // Log invalid files List <AdfFileInfo> invalidFiles = allFiles.Where(x => !x.IsValid).ToList(); if (invalidFiles.Any()) { logger.Write("The following files found in the output folder will not be published:"); foreach (AdfFileInfo invalidFile in invalidFiles) { logger.Write(invalidFile.FileName); } } DataFactoryManagementClient client = GetDataFactoryManagementClient(); // Deploy Package Zips before deploying ADF JSON files List <AdfFileInfo> pipelines = validFiles.Where(x => x.FileType == FileType.Pipeline).ToList(); List <CustomActivityPackageInfo> packages = pipelines.SelectMany(x => x.CustomActivityPackages).Distinct().ToList(); if (packages.Any()) { result &= await DeployCustomActivities(packages, validFiles, outputFolder); } List <AdfFileInfo> linkedServices = validFiles.Where(x => x.FileType == FileType.LinkedService).ToList(); if (linkedServices.Any()) { logger.Write(string.Empty); logger.Write("Deploying LinkedServices", "Black"); // Deploy non batch linked services first var linkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >(); foreach (var linkedService in linkedServices.Where(x => !x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase))) { linkedServiceTaskDict.Add(linkedService.Name, client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, linkedService.Name, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(linkedService.FileContents))); } foreach (var item in linkedServiceTaskDict) { try { LinkedServiceCreateOrUpdateResponse response = await item.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green"); } else { logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (Exception e) { logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red"); logger.WriteError(e); result = false; } } // Deploy batch linked services next var batchLinkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >(); foreach (var batchLinkedService in linkedServices.Where(x => x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase))) { batchLinkedServiceTaskDict.Add(batchLinkedService.Name, client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, batchLinkedService.Name, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(batchLinkedService.FileContents))); } foreach (var item in batchLinkedServiceTaskDict) { try { LinkedServiceCreateOrUpdateResponse response = await item.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green"); } else { logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (Exception e) { logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red"); logger.WriteError(e); result = false; } } } List <AdfFileInfo> tables = validFiles.Where(x => x.FileType == FileType.Table).ToList(); if (tables.Any()) { logger.Write(string.Empty); logger.Write("Deploying tables", "Black"); // Deploy tables next var tableTaskDict = new Dictionary <string, Task <DatasetCreateOrUpdateResponse> >(); foreach (AdfFileInfo adfJsonFile in tables) { try { Task <DatasetCreateOrUpdateResponse> tableTask = client.Datasets.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, adfJsonFile.Name, new DatasetCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents)); tableTaskDict.Add(adfJsonFile.Name, tableTask); } catch (Exception e) { logger.Write($"An error occurred uploading table '{adfJsonFile.Name}': " + e.Message, "Red"); logger.WriteError(e); result = false; } } foreach (var task in tableTaskDict) { try { DatasetCreateOrUpdateResponse response = await task.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Table '{task.Key}' uploaded successfully.", "Green"); } else { logger.Write( $"Table '{task.Key}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (CloudException ex) { if (ex.Error.Code == "TableAvailabilityUpdateNotSupported") { logger.Write($"It looks like you are trying to change the availability for the Table '{task.Key}'. Currently this is not supported by ADF so as work around you should delete the dataset and related pipleline in the Data Factory '{dataFactoryName}' and run the publish again.", "Red"); } else { logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red"); } logger.WriteError(ex); result = false; } catch (Exception ex) { logger.WriteError(ex); logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red"); result = false; } } } if (pipelines.Any()) { logger.Write(string.Empty); logger.Write("Deploying pipelines", "Black"); // Deploy pipelines last var pipelineTaskDict = new Dictionary <string, Task <PipelineCreateOrUpdateResponse> >(); foreach (AdfFileInfo adfJsonFile in pipelines) { pipelineTaskDict.Add(adfJsonFile.Name, client.Pipelines.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, adfJsonFile.Name, new PipelineCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents))); } foreach (var item in pipelineTaskDict) { try { PipelineCreateOrUpdateResponse response = await item.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Pipeline '{response.Pipeline.Name}' uploaded successfully.", "Green"); } else { logger.Write($"Pipeline '{response.Pipeline.Name}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (Exception e) { logger.WriteError(e); logger.Write($"An error occurred uploading pipeline '{item.Key}': " + e.Message, "Red"); result = false; } } } return(result); }