public async Task <LinkedServiceCreateOrUpdateResponse> GetCreateOrUpdateStatusAsync(
string operationStatusLink,
CancellationToken cancellationToken)
{
Ensure.IsNotNull(operationStatusLink, "operationStatusLink");
Core.Models.LinkedServiceCreateOrUpdateResponse internalResponse =
await this.Client.InternalClient.LinkedServices.GetCreateOrUpdateStatusAsync(
operationStatusLink,
cancellationToken);
var response = new LinkedServiceCreateOrUpdateResponse(internalResponse, this.Client);
if (response.LinkedService != null && response.LinkedService.Properties != null &&
response.LinkedService.Properties.ProvisioningState != null)
{
response.Status = response.LinkedService.Properties.ProvisioningState.ToOperationStatus();
}
return(response);
}
/// <summary>
/// Create or update a linked service.
/// </summary>
/// <param name='resourceGroupName'>
/// Required. The resource group name of the linked service.
/// </param>
/// <param name='workspaceName'>
/// Required. The name of the parent workspace that will contain the
/// linked service
/// </param>
/// <param name='parameters'>
/// Required. The parameters required to create or update a linked
/// service.
/// </param>
/// <param name='cancellationToken'>
/// Cancellation token.
/// </param>
/// <returns>
/// The create or update linked service operation response.
/// </returns>
public async Task <LinkedServiceCreateOrUpdateResponse> CreateOrUpdateAsync(string resourceGroupName, string workspaceName, LinkedServiceCreateOrUpdateParameters parameters, CancellationToken cancellationToken)
{
// Validate
if (resourceGroupName == null)
{
throw new ArgumentNullException("resourceGroupName");
}
if (workspaceName == null)
{
throw new ArgumentNullException("workspaceName");
}
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
if (parameters.Name == null)
{
throw new ArgumentNullException("parameters.Name");
}
if (parameters.Properties == null)
{
throw new ArgumentNullException("parameters.Properties");
}
if (parameters.Properties.ResourceId == null)
{
throw new ArgumentNullException("parameters.Properties.ResourceId");
}
// Tracing
bool shouldTrace = TracingAdapter.IsEnabled;
string invocationId = null;
if (shouldTrace)
{
invocationId = TracingAdapter.NextInvocationId.ToString();
Dictionary <string, object> tracingParameters = new Dictionary <string, object>();
tracingParameters.Add("resourceGroupName", resourceGroupName);
tracingParameters.Add("workspaceName", workspaceName);
tracingParameters.Add("parameters", parameters);
TracingAdapter.Enter(invocationId, this, "CreateOrUpdateAsync", tracingParameters);
}
// Construct URL
string url = "";
url = url + "/subscriptions/";
if (this.Client.Credentials.SubscriptionId != null)
{
url = url + Uri.EscapeDataString(this.Client.Credentials.SubscriptionId);
}
url = url + "/resourcegroups/";
url = url + Uri.EscapeDataString(resourceGroupName);
url = url + "/providers/Microsoft.OperationalInsights/workspaces/";
url = url + Uri.EscapeDataString(workspaceName);
url = url + "/linkedServices/";
url = url + Uri.EscapeDataString(parameters.Name);
List <string> queryParameters = new List <string>();
queryParameters.Add("api-version=2015-11-01-preview");
if (queryParameters.Count > 0)
{
url = url + "?" + string.Join("&", queryParameters);
}
string baseUrl = this.Client.BaseUri.AbsoluteUri;
// Trim '/' character from the end of baseUrl and beginning of url.
if (baseUrl[baseUrl.Length - 1] == '/')
{
baseUrl = baseUrl.Substring(0, baseUrl.Length - 1);
}
if (url[0] == '/')
{
url = url.Substring(1);
}
url = baseUrl + "/" + url;
url = url.Replace(" ", "%20");
// Create HTTP transport objects
HttpRequestMessage httpRequest = null;
try
{
httpRequest = new HttpRequestMessage();
httpRequest.Method = HttpMethod.Put;
httpRequest.RequestUri = new Uri(url);
// Set Headers
httpRequest.Headers.Add("x-ms-client-request-id", Guid.NewGuid().ToString());
// Set Credentials
cancellationToken.ThrowIfCancellationRequested();
await this.Client.Credentials.ProcessHttpRequestAsync(httpRequest, cancellationToken).ConfigureAwait(false);
// Serialize Request
string requestContent = null;
JToken requestDoc = null;
JObject linkedServiceCreateOrUpdateParametersValue = new JObject();
requestDoc = linkedServiceCreateOrUpdateParametersValue;
linkedServiceCreateOrUpdateParametersValue["name"] = parameters.Name;
JObject propertiesValue = new JObject();
linkedServiceCreateOrUpdateParametersValue["properties"] = propertiesValue;
propertiesValue["resourceId"] = parameters.Properties.ResourceId;
requestContent = requestDoc.ToString(Newtonsoft.Json.Formatting.Indented);
httpRequest.Content = new StringContent(requestContent, Encoding.UTF8);
httpRequest.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
// Send Request
HttpResponseMessage httpResponse = null;
try
{
if (shouldTrace)
{
TracingAdapter.SendRequest(invocationId, httpRequest);
}
cancellationToken.ThrowIfCancellationRequested();
httpResponse = await this.Client.HttpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
if (shouldTrace)
{
TracingAdapter.ReceiveResponse(invocationId, httpResponse);
}
HttpStatusCode statusCode = httpResponse.StatusCode;
if (statusCode != HttpStatusCode.OK && statusCode != HttpStatusCode.Created)
{
cancellationToken.ThrowIfCancellationRequested();
CloudException ex = CloudException.Create(httpRequest, requestContent, httpResponse, await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false));
if (shouldTrace)
{
TracingAdapter.Error(invocationId, ex);
}
throw ex;
}
// Create Result
LinkedServiceCreateOrUpdateResponse result = null;
// Deserialize Response
if (statusCode == HttpStatusCode.OK || statusCode == HttpStatusCode.Created)
{
cancellationToken.ThrowIfCancellationRequested();
string responseContent = await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
result = new LinkedServiceCreateOrUpdateResponse();
JToken responseDoc = null;
if (string.IsNullOrEmpty(responseContent) == false)
{
responseDoc = JToken.Parse(responseContent);
}
if (responseDoc != null && responseDoc.Type != JTokenType.Null)
{
LinkedService linkedServiceInstance = new LinkedService();
result.LinkedService = linkedServiceInstance;
JToken idValue = responseDoc["id"];
if (idValue != null && idValue.Type != JTokenType.Null)
{
string idInstance = ((string)idValue);
linkedServiceInstance.Id = idInstance;
}
JToken nameValue = responseDoc["name"];
if (nameValue != null && nameValue.Type != JTokenType.Null)
{
string nameInstance = ((string)nameValue);
linkedServiceInstance.Name = nameInstance;
}
JToken typeValue = responseDoc["type"];
if (typeValue != null && typeValue.Type != JTokenType.Null)
{
string typeInstance = ((string)typeValue);
linkedServiceInstance.Type = typeInstance;
}
JToken propertiesValue2 = responseDoc["properties"];
if (propertiesValue2 != null && propertiesValue2.Type != JTokenType.Null)
{
LinkedServiceProperties propertiesInstance = new LinkedServiceProperties();
linkedServiceInstance.Properties = propertiesInstance;
JToken resourceIdValue = propertiesValue2["resourceId"];
if (resourceIdValue != null && resourceIdValue.Type != JTokenType.Null)
{
string resourceIdInstance = ((string)resourceIdValue);
propertiesInstance.ResourceId = resourceIdInstance;
}
}
}
}
result.StatusCode = statusCode;
if (httpResponse.Headers.Contains("x-ms-request-id"))
{
result.RequestId = httpResponse.Headers.GetValues("x-ms-request-id").FirstOrDefault();
}
if (shouldTrace)
{
TracingAdapter.Exit(invocationId, result);
}
return(result);
}
finally
{
if (httpResponse != null)
{
httpResponse.Dispose();
}
}
}
finally
{
if (httpRequest != null)
{
httpRequest.Dispose();
}
}
}
/// <summary>
/// Deploys the specified ADF files and custom activity packages to Azure.
/// </summary>
/// <param name="filesToProcess">The files to process.</param>
/// <param name="outputFolder">The output folder which contains the files and custom activity zips.</param>
/// <param name="deployConfig">The deployment configuration information.</param>
public async Task <bool> Deploy(List <string> filesToProcess, string outputFolder, DeployConfigInfo deployConfig = null)
{
bool result = true;
logger.Write(string.Empty);
logger.Write($"Getting all ADF resources to deploy to Azure from output folder '{outputFolder}'", "Black");
List <Task <AdfFileInfo> > allFilesTasks = filesToProcess.Select(async x => await adfFileHelper.GetFileInfo(x, deployConfig)).ToList();
List <AdfFileInfo> allFiles = new List <AdfFileInfo>();
foreach (var allFilesTask in allFilesTasks)
{
allFiles.Add(await allFilesTask);
}
List <AdfFileInfo> validFiles = allFiles.Where(x => x.IsValid).ToList();
if (!validFiles.Any())
{
logger.Write($"No valid ADF files found in '{outputFolder}'", "Red");
return(false);
}
logger.Write($"{validFiles.Count} file{(validFiles.Count == 1 ? string.Empty : "s")} retreived");
logger.Write(string.Empty);
logger.Write($"Begin deploying ADF resources to '{dataFactoryName}'", "Black");
// Log invalid files
List <AdfFileInfo> invalidFiles = allFiles.Where(x => !x.IsValid).ToList();
if (invalidFiles.Any())
{
logger.Write("The following files found in the output folder will not be published:");
foreach (AdfFileInfo invalidFile in invalidFiles)
{
logger.Write(invalidFile.FileName);
}
}
DataFactoryManagementClient client = GetDataFactoryManagementClient();
// Deploy Package Zips before deploying ADF JSON files
List <AdfFileInfo> pipelines = validFiles.Where(x => x.FileType == FileType.Pipeline).ToList();
List <CustomActivityPackageInfo> packages = pipelines.SelectMany(x => x.CustomActivityPackages).Distinct().ToList();
if (packages.Any())
{
result &= await DeployCustomActivities(packages, validFiles, outputFolder);
}
List <AdfFileInfo> linkedServices = validFiles.Where(x => x.FileType == FileType.LinkedService).ToList();
if (linkedServices.Any())
{
logger.Write(string.Empty);
logger.Write("Deploying LinkedServices", "Black");
// Deploy non batch linked services first
var linkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >();
foreach (var linkedService in linkedServices.Where(x => !x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase)))
{
linkedServiceTaskDict.Add(linkedService.Name,
client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName,
dataFactoryName, linkedService.Name,
new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(linkedService.FileContents)));
}
foreach (var item in linkedServiceTaskDict)
{
try
{
LinkedServiceCreateOrUpdateResponse response = await item.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green");
}
else
{
logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red");
result = false;
}
}
catch (Exception e)
{
logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red");
logger.WriteError(e);
result = false;
}
}
// Deploy batch linked services next
var batchLinkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >();
foreach (var batchLinkedService in linkedServices.Where(x => x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase)))
{
batchLinkedServiceTaskDict.Add(batchLinkedService.Name,
client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName,
dataFactoryName, batchLinkedService.Name,
new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(batchLinkedService.FileContents)));
}
foreach (var item in batchLinkedServiceTaskDict)
{
try
{
LinkedServiceCreateOrUpdateResponse response = await item.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green");
}
else
{
logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red");
result = false;
}
}
catch (Exception e)
{
logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red");
logger.WriteError(e);
result = false;
}
}
}
List <AdfFileInfo> tables = validFiles.Where(x => x.FileType == FileType.Table).ToList();
if (tables.Any())
{
logger.Write(string.Empty);
logger.Write("Deploying tables", "Black");
// Deploy tables next
var tableTaskDict = new Dictionary <string, Task <DatasetCreateOrUpdateResponse> >();
foreach (AdfFileInfo adfJsonFile in tables)
{
try
{
Task <DatasetCreateOrUpdateResponse> tableTask = client.Datasets.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName,
adfJsonFile.Name, new DatasetCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents));
tableTaskDict.Add(adfJsonFile.Name, tableTask);
}
catch (Exception e)
{
logger.Write($"An error occurred uploading table '{adfJsonFile.Name}': " + e.Message, "Red");
logger.WriteError(e);
result = false;
}
}
foreach (var task in tableTaskDict)
{
try
{
DatasetCreateOrUpdateResponse response = await task.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Table '{task.Key}' uploaded successfully.", "Green");
}
else
{
logger.Write(
$"Table '{task.Key}' did not upload successfully. Response status: {response.Status}",
"Red");
result = false;
}
}
catch (CloudException ex)
{
if (ex.Error.Code == "TableAvailabilityUpdateNotSupported")
{
logger.Write($"It looks like you are trying to change the availability for the Table '{task.Key}'. Currently this is not supported by ADF so as work around you should delete the dataset and related pipleline in the Data Factory '{dataFactoryName}' and run the publish again.", "Red");
}
else
{
logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red");
}
logger.WriteError(ex);
result = false;
}
catch (Exception ex)
{
logger.WriteError(ex);
logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red");
result = false;
}
}
}
if (pipelines.Any())
{
logger.Write(string.Empty);
logger.Write("Deploying pipelines", "Black");
// Deploy pipelines last
var pipelineTaskDict = new Dictionary <string, Task <PipelineCreateOrUpdateResponse> >();
foreach (AdfFileInfo adfJsonFile in pipelines)
{
pipelineTaskDict.Add(adfJsonFile.Name, client.Pipelines.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName,
adfJsonFile.Name, new PipelineCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents)));
}
foreach (var item in pipelineTaskDict)
{
try
{
PipelineCreateOrUpdateResponse response = await item.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Pipeline '{response.Pipeline.Name}' uploaded successfully.", "Green");
}
else
{
logger.Write($"Pipeline '{response.Pipeline.Name}' did not upload successfully. Response status: {response.Status}", "Red");
result = false;
}
}
catch (Exception e)
{
logger.WriteError(e);
logger.Write($"An error occurred uploading pipeline '{item.Key}': " + e.Message, "Red");
result = false;
}
}
}
return(result);
}
