public LinkedServiceResource ToSdkObject()
{
LinkedService linkedService = this.Properties?.ToSdkObject();
LinkedServiceResource linkedServiceResource = new LinkedServiceResource(linkedService);
return(linkedServiceResource);
}
public static void CreateOrUpdate(string resourceName, LinkedServiceResource linkedServiceResource)
{
if (Interface != null)
{
Interface.LinkedServices.CreateOrUpdate(resourceGroup, dataFactoryName, resourceName, linkedServiceResource);
}
}
public async Task LinkedServiceSample()
{
#region Snippet:CreateLinkedServiceClient
// Replace the string below with your actual endpoint url.
string endpoint = "<my-endpoint-url>";
/*@@*/ endpoint = TestEnvironment.EndpointUrl;
LinkedServiceClient client = new LinkedServiceClient(endpoint: new Uri(endpoint), credential: new DefaultAzureCredential());
#endregion
#region Snippet:CreateLinkedService
LinkedServiceResource serviceResource = new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/"));
LinkedServiceCreateOrUpdateLinkedServiceOperation operation = client.StartCreateOrUpdateLinkedService("MyLinkedService", serviceResource);
Response <LinkedServiceResource> createdService = await operation.WaitForCompletionAsync();
#endregion
#region Snippet:RetrieveLinkedService
LinkedServiceResource retrievedService = client.GetLinkedService("MyLinkedService");
#endregion
#region Snippet:ListLinkedServices
Pageable <LinkedServiceResource> linkedServices = client.GetLinkedServicesByWorkspace();
foreach (LinkedServiceResource linkedService in linkedServices)
{
System.Console.WriteLine(linkedService.Name);
}
#endregion
#region Snippet:DeleteLinkedService
client.StartDeleteLinkedService("MyLinkedService");
#endregion
}
public LinkedServiceResource CreateOrUpdateLinkedService(string linkedServiceName, string rawJsonContent)
{
LinkedServiceResource linkedService = JsonConvert.DeserializeObject <LinkedServiceResource>(rawJsonContent);
var operation = _linkedServiceClient.StartCreateOrUpdateLinkedService(linkedServiceName, linkedService);
return(operation.Poll().Value);
}
public void CreateLinkedService()
{
#region Snippet:CreateLinkedService
var operation = LinkedServiceClient.StartCreateOrUpdateLinkedService("MyLinkedService", new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/")));
LinkedServiceResource linkedService = operation.WaitForCompletionAsync().ConfigureAwait(true).GetAwaiter().GetResult();
#endregion
}
public static async ValueTask <LinkedServiceResource> CreateResource(LinkedServiceClient client, TestRecording recording)
{
string linkedServiceName = recording.GenerateId("LinkedService", 16);
LinkedServiceResource resource = new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/"));
LinkedServiceCreateOrUpdateLinkedServiceOperation operation = await client.StartCreateOrUpdateLinkedServiceAsync(linkedServiceName, resource);
return(await operation.WaitForCompletionAsync());
}
public void LinkedServices_Update()
{
RunTest("LinkedServices_Update", (example, client, responseCode) =>
{
LinkedServiceResource resource = client.LinkedServices.CreateOrUpdate(RGN(example), FN(example), LSN(example), LSR(example, client));
CheckResponseBody(example, client, responseCode, resource);
});
}
public void LinkedServices_Get()
{
RunTest("LinkedServices_Get", (example, client, responseCode) =>
{
LinkedServiceResource resource = client.LinkedServices.Get(RGN(example), FN(example), LSN(example));
CheckResponseBody(example, client, responseCode, resource);
});
}
public async Task TestCreateLinkedService()
{
string linkedServiceName = Recording.GenerateName("LinkedSercive");
LinkedServiceCreateOrUpdateLinkedServiceOperation operation = await LinkedServiceClient.StartCreateOrUpdateLinkedServiceAsync(linkedServiceName, new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/")));
LinkedServiceResource linkedService = await operation.WaitForCompletionAsync();
Assert.AreEqual(linkedServiceName, linkedService.Name);
}
internal CreateDataFlowDebugSessionRequest(string dataFlowName, string existingClusterId, int?clusterTimeout, string newClusterName, string newClusterNodeType, LinkedServiceResource dataBricksLinkedService)
{
DataFlowName = dataFlowName;
ExistingClusterId = existingClusterId;
ClusterTimeout = clusterTimeout;
NewClusterName = newClusterName;
NewClusterNodeType = newClusterNodeType;
DataBricksLinkedService = dataBricksLinkedService;
}
public PSLinkedServiceResource(LinkedServiceResource linkedServiceResource, string workspaceName)
: base(linkedServiceResource?.Id,
linkedServiceResource?.Name,
linkedServiceResource?.Type,
linkedServiceResource?.Etag)
{
this.WorkspaceName = workspaceName;
this.Properties = new PSLinkedService(linkedServiceResource?.Properties);
}
private static void createDataWarehouse(DataFactoryManagementClient client)
{
var IR = new IntegrationRuntimeReference(DatosGrales.azureIntegrationRuntime);
var conStr = new SecureString(DatosGrales.warehouseConnectionString);
LinkedServiceResource SqlServerLinkedServiceWarehouse = new LinkedServiceResource(
new AzureSqlDWLinkedService(conStr, null, IR, "Sql Warehouse - " + DatosGrales.nombreBDWarehouse));
client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, "SqlServerLinkedService-Warehouse", SqlServerLinkedServiceWarehouse);
}
public async Task TestDeleteLinkedService()
{
LinkedServiceClient client = CreateClient();
LinkedServiceResource resource = await DisposableLinkedService.CreateResource(client, this.Recording);
LinkedServiceDeleteLinkedServiceOperation operation = await client.StartDeleteLinkedServiceAsync(resource.Name);
await operation.WaitAndAssertSuccessfulCompletion();
}
public async Task TestGetLinkedService()
{
await foreach (var expectedLinkedService in LinkedServiceClient.GetLinkedServicesByWorkspaceAsync())
{
LinkedServiceResource actualLinkedService = await LinkedServiceClient.GetLinkedServiceAsync(expectedLinkedService.Name);
Assert.AreEqual(expectedLinkedService.Name, actualLinkedService.Name);
Assert.AreEqual(expectedLinkedService.Id, actualLinkedService.Id);
}
}
private static void createAzureSSIS(DataFactoryManagementClient client)
{
var IR = new IntegrationRuntimeReference(DatosGrales.onPremiseIntegrationRuntime);
var pass = new SecureString(DatosGrales.passwordWarehouse);
LinkedServiceResource SqlServerLinkedServiceSSIS = new LinkedServiceResource(
new SqlServerLinkedService(
new SecureString(@"Data Source=sqlsrvbi00.database.windows.net;Initial Catalog=SSISDB;Integrated Security=False"), null, IR, "Sql que hostea el SSIS", DatosGrales.usuarioWarehouse, pass));
client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, DatosGrales.linkedServiceSSIS, SqlServerLinkedServiceSSIS);
}
private static void createSQLServers(DataFactoryManagementClient client)
{
var IR = new IntegrationRuntimeReference(DatosGrales.onPremiseIntegrationRuntime);
string nombreBD = DatosGrales.nombreBD;
var pass = new SecureString(DatosGrales.passwordOnPremise);
LinkedServiceResource SqlServerLinkedServiceClaim = new LinkedServiceResource(
new SqlServerLinkedService(
new SecureString(@"Data Source=ROW2K12SQL11;Initial Catalog=" + nombreBD + ";Integrated Security=True"), null, IR, "Sql Local - " + nombreBD, DatosGrales.usuarioOnPremise, pass));
client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, "SqlServerLinkedService-" + nombreBD, SqlServerLinkedServiceClaim);
}
public PSLinkedService(LinkedServiceResource linkedService, string resourceGroupName, string factoryName)
{
if (linkedService == null)
{
throw new ArgumentNullException("linkedService");
}
this.linkedService = linkedService;
this.ResourceGroupName = resourceGroupName;
this.DataFactoryName = factoryName;
}
public virtual PSLinkedService GetLinkedService(string resourceGroupName, string dataFactoryName,
string linkedServiceName)
{
LinkedServiceResource response = this.DataFactoryManagementClient.LinkedServices.Get(resourceGroupName, dataFactoryName,
linkedServiceName);
if (response == null)
{
return(null);
}
return(new PSLinkedService(response, resourceGroupName, dataFactoryName));
}
public LinkedServiceResource CreateOrUpdateLinkedService(string linkedServiceName, string rawJsonContent)
{
PSLinkedServiceResource psLinkedService = JsonConvert.DeserializeObject <PSLinkedServiceResource>(rawJsonContent, Settings);
LinkedServiceResource linkedService = psLinkedService.ToSdkObject();
var operation = _linkedServiceClient.StartCreateOrUpdateLinkedService(linkedServiceName, linkedService);
while (!operation.HasValue)
{
operation.UpdateStatus();
}
return(operation.Value);
}
internal static LinkedServiceResource GetLinkedServiceResource(string description)
{
LinkedServiceResource resource = new LinkedServiceResource
{
Properties = new AzureDataLakeStoreLinkedService()
{
Description = description,
DataLakeStoreUri = "adl://test.azuredatalakestore.net/"
}
};
return(resource);
}
private LinkedServiceResource GetLinkedServiceResource(string description)
{
LinkedServiceResource resource = new LinkedServiceResource
{
Properties = new AzureStorageLinkedService
{
Description = description,
ConnectionString = new SecureString(string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", secrets.StorageAccountName, secrets.StorageAccountKey))
}
};
return(resource);
}
private static void createDataLake(DataFactoryManagementClient client)
{
AzureDataLakeStoreLinkedService DataLakeLinkedService = new AzureDataLakeStoreLinkedService();
DataLakeLinkedService.DataLakeStoreUri = "https://datalakebi00.azuredatalakestore.net/webhdfs/v1";
DataLakeLinkedService.ServicePrincipalId = DatosGrales.applicationId;
DataLakeLinkedService.ServicePrincipalKey = new SecureString(DatosGrales.authenticationKey);
DataLakeLinkedService.ResourceGroupName = DatosGrales.resourceGroup;
DataLakeLinkedService.SubscriptionId = DatosGrales.subscriptionId;
DataLakeLinkedService.Tenant = DatosGrales.tenantID;
LinkedServiceResource DataLakeResource = new LinkedServiceResource(DataLakeLinkedService);
DataLakeResource.Properties.ConnectVia = new IntegrationRuntimeReference(DatosGrales.azureIntegrationRuntime);
client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, DatosGrales.linkedServiceLake, DataLakeResource);
}
public static void AddAzureSqlDatabaseLinkedService(
this DataFactoryManagementClient client,
AzureConfig config,
string sqlDatabaseConnectionString,
string sqlDatabaseLinkedServiceName)
{
Console.WriteLine($"Creating linked service {sqlDatabaseLinkedServiceName}...");
var storageLinkedService = new LinkedServiceResource(
new AzureSqlDatabaseLinkedService
{
ConnectionString = sqlDatabaseConnectionString
}
);
client.LinkedServices.CreateOrUpdate(config.ResourceGroup, config.DataFactoryName, sqlDatabaseLinkedServiceName, storageLinkedService);
Console.WriteLine(SafeJsonConvert.SerializeObject(storageLinkedService, client.SerializationSettings));
}
public async Task TestGetLinkedService()
{
LinkedServiceClient client = CreateClient();
await using DisposableLinkedService service = await DisposableLinkedService.Create(client, this.Recording);
IList <LinkedServiceResource> services = await client.GetLinkedServicesByWorkspaceAsync().ToListAsync();
Assert.GreaterOrEqual(services.Count, 1);
foreach (var expectedLinkedService in services)
{
LinkedServiceResource actualLinkedService = await client.GetLinkedServiceAsync(expectedLinkedService.Name);
Assert.AreEqual(expectedLinkedService.Name, actualLinkedService.Name);
Assert.AreEqual(expectedLinkedService.Id, actualLinkedService.Id);
}
}
public static void CreateBlobLS(DataFactoryManagementClient client)
{
Console.WriteLine("Creating linked service " + Constants.storageLinkedServiceName + "...");
LinkedServiceResource storageLinkedService = new LinkedServiceResource(
new AzureStorageLinkedService
{
ConnectionString = new SecureString(
"DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount +
";AccountKey=" + Constants.storageKey)
}
);
client.LinkedServices.CreateOrUpdate(
Constants.resourceGroup, Constants.dataFactoryName, Constants.storageLinkedServiceName, storageLinkedService);
//Console.WriteLine(SafeJsonConvert.SerializeObject(
// storageLinkedService, client.SerializationSettings));
}
public async Task <ActionResult <LinkedServiceResource> > AddDataSource2Async(Guid engineId,
string dataSourceName, [FromBody] LinkedService linkedService)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
var accessToken = await this.authProvider.GetAccessTokenForAppManagementAsync().ConfigureAwait(false);
ServiceClientCredentials tokenCredentials = new TokenCredentials(accessToken);
DataFactoryManagementClient client = new DataFactoryManagementClient(tokenCredentials)
{
SubscriptionId = this.options.SubscriptionId
};
// Create the linked service resource
LinkedServiceResource linkedServiceResource = new LinkedServiceResource(linkedService);
string sensitiveString = linkedServiceResource.Properties switch
{
AzureStorageLinkedService azureStorageLinkedService => azureStorageLinkedService.ConnectionString.ToString(),
_ => null
};
if (!string.IsNullOrEmpty(sensitiveString))
{
// Save the connection string to KeyVault
await keyVaultsController.SetKeyVaultSecret(engineId, dataSourceName,
new YKeyVaultSecretPayload { Key = dataSourceName, Value = sensitiveString });
}
var newLinkedServiceResourceCreated = await client.LinkedServices.CreateOrUpdateAsync(engine.ResourceGroupName, engine.FactoryName, dataSourceName, linkedServiceResource);
return(newLinkedServiceResourceCreated);
}
public void LinkedService_WebLinkedServiceSDKSample()
{
string password = "******";
LinkedServiceResource linkedServiceResource = new LinkedServiceResource
{
Properties = new WebLinkedService
{
TypeProperties = new WebBasicAuthentication
{
Url = "https://localhost",
Username = "******",
Password = new SecureString(password)
}
}
};
var handler = new RecordedDelegatingHandler();
var client = this.CreateWorkflowClient(handler);
var json = SafeJsonConvert.SerializeObject(linkedServiceResource, client.SerializationSettings);
Assert.Contains(password, json);
}
public async Task TestRenameLinkedService()
{
LinkedServiceClient client = CreateClient();
LinkedServiceResource resource = await DisposableLinkedService.CreateResource(client, Recording);
string newLinkedServiceName = Recording.GenerateId("LinkedService2", 16);
LinkedServiceRenameLinkedServiceOperation renameOperation = await client.StartRenameLinkedServiceAsync(resource.Name, new ArtifactRenameRequest()
{
NewName = newLinkedServiceName
});
await renameOperation.WaitForCompletionResponseAsync();
LinkedServiceResource service = await client.GetLinkedServiceAsync(newLinkedServiceName);
Assert.AreEqual(newLinkedServiceName, service.Name);
LinkedServiceDeleteLinkedServiceOperation operation = await client.StartDeleteLinkedServiceAsync(newLinkedServiceName);
await operation.WaitForCompletionResponseAsync();
}
static void Main(string[] args)
{
// Set variables
string tenantID = "<your tenant ID>";
string applicationId = "<your application ID>";
string authenticationKey = "<your authentication key for the application>";
string subscriptionId = "<your subscription ID where the data factory resides>";
string resourceGroup = "<your resource group where the data factory resides>";
string region = "<the location of your resource group>";
string dataFactoryName =
"<specify the name of data factory to create. It must be globally unique.>";
string storageAccount = "<your storage account name to copy data>";
string storageKey = "<your storage account key>";
// specify the container and input folder from which all files
// need to be copied to the output folder.
string inputBlobPath =
"<path to existing blob(s) to copy data from, e.g. containername/inputdir>";
//specify the contains and output folder where the files are copied
string outputBlobPath =
"<the blob path to copy data to, e.g. containername/outputdir>";
// name of the Azure Storage linked service, blob dataset, and the pipeline
string storageLinkedServiceName = "AzureStorageLinkedService";
string blobDatasetName = "BlobDataset";
string pipelineName = "Adfv2QuickStartPipeline";
// Authenticate and create a data factory management client
var context = new AuthenticationContext("https://login.microsoftonline.com/" + tenantID);
ClientCredential cc = new ClientCredential(applicationId, authenticationKey);
AuthenticationResult result = context.AcquireTokenAsync(
"https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(result.AccessToken);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
// Create a data factory
Console.WriteLine("Creating data factory " + dataFactoryName + "...");
Factory dataFactory = new Factory
{
Location = region,
Identity = new FactoryIdentity()
};
client.Factories.CreateOrUpdate(resourceGroup, dataFactoryName, dataFactory);
Console.WriteLine(
SafeJsonConvert.SerializeObject(dataFactory, client.SerializationSettings));
while (client.Factories.Get(resourceGroup, dataFactoryName).ProvisioningState ==
"PendingCreation")
{
System.Threading.Thread.Sleep(1000);
}
// Create an Azure Storage linked service
Console.WriteLine("Creating linked service " + storageLinkedServiceName + "...");
LinkedServiceResource storageLinkedService = new LinkedServiceResource(
new AzureStorageLinkedService
{
ConnectionString = new SecureString(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccount +
";AccountKey=" + storageKey)
}
);
client.LinkedServices.CreateOrUpdate(
resourceGroup, dataFactoryName, storageLinkedServiceName, storageLinkedService);
Console.WriteLine(SafeJsonConvert.SerializeObject(
storageLinkedService, client.SerializationSettings));
// Create an Azure Blob dataset
Console.WriteLine("Creating dataset " + blobDatasetName + "...");
DatasetResource blobDataset = new DatasetResource(
new AzureBlobDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = storageLinkedServiceName
},
FolderPath = new Expression {
Value = "@{dataset().path}"
},
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "path", new ParameterSpecification {
Type = ParameterType.String
} }
}
}
);
client.Datasets.CreateOrUpdate(
resourceGroup, dataFactoryName, blobDatasetName, blobDataset);
Console.WriteLine(
SafeJsonConvert.SerializeObject(blobDataset, client.SerializationSettings));
// Create a pipeline with a copy activity
Console.WriteLine("Creating pipeline " + pipelineName + "...");
PipelineResource pipeline = new PipelineResource
{
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "inputPath", new ParameterSpecification {
Type = ParameterType.String
} },
{ "outputPath", new ParameterSpecification {
Type = ParameterType.String
} }
},
Activities = new List <Activity>
{
new CopyActivity
{
Name = "CopyFromBlobToBlob",
Inputs = new List <DatasetReference>
{
new DatasetReference()
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.inputPath" }
}
}
},
Outputs = new List <DatasetReference>
{
new DatasetReference
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.outputPath" }
}
}
},
Source = new BlobSource {
},
Sink = new BlobSink {
}
}
}
};
client.Pipelines.CreateOrUpdate(resourceGroup, dataFactoryName, pipelineName, pipeline);
Console.WriteLine(SafeJsonConvert.SerializeObject(pipeline, client.SerializationSettings));
// Create a pipeline run
Console.WriteLine("Creating pipeline run...");
Dictionary <string, object> parameters = new Dictionary <string, object>
{
{ "inputPath", inputBlobPath },
{ "outputPath", outputBlobPath }
};
CreateRunResponse runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, dataFactoryName, pipelineName, parameters: parameters
).Result.Body;
Console.WriteLine("Pipeline run ID: " + runResponse.RunId);
// Monitor the pipeline run
Console.WriteLine("Checking pipeline run status...");
PipelineRun pipelineRun;
while (true)
{
pipelineRun = client.PipelineRuns.Get(
resourceGroup, dataFactoryName, runResponse.RunId);
Console.WriteLine("Status: " + pipelineRun.Status);
if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
// Check the copy activity run details
Console.WriteLine("Checking copy activity run details...");
RunFilterParameters filterParams = new RunFilterParameters(
DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
resourceGroup, dataFactoryName, runResponse.RunId, filterParams);
if (pipelineRun.Status == "Succeeded")
{
Console.WriteLine(queryResponse.Value.First().Output);
}
else
{
Console.WriteLine(queryResponse.Value.First().Error);
}
Console.WriteLine("\nPress any key to exit...");
Console.ReadKey();
}
public PSLinkedService()
{
linkedService = new LinkedServiceResource();
}
