public LinkedServiceResource ToSdkObject() { LinkedService linkedService = this.Properties?.ToSdkObject(); LinkedServiceResource linkedServiceResource = new LinkedServiceResource(linkedService); return(linkedServiceResource); }
public static void CreateOrUpdate(string resourceName, LinkedServiceResource linkedServiceResource) { if (Interface != null) { Interface.LinkedServices.CreateOrUpdate(resourceGroup, dataFactoryName, resourceName, linkedServiceResource); } }
public async Task LinkedServiceSample() { #region Snippet:CreateLinkedServiceClient // Replace the string below with your actual endpoint url. string endpoint = "<my-endpoint-url>"; /*@@*/ endpoint = TestEnvironment.EndpointUrl; LinkedServiceClient client = new LinkedServiceClient(endpoint: new Uri(endpoint), credential: new DefaultAzureCredential()); #endregion #region Snippet:CreateLinkedService LinkedServiceResource serviceResource = new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/")); LinkedServiceCreateOrUpdateLinkedServiceOperation operation = client.StartCreateOrUpdateLinkedService("MyLinkedService", serviceResource); Response <LinkedServiceResource> createdService = await operation.WaitForCompletionAsync(); #endregion #region Snippet:RetrieveLinkedService LinkedServiceResource retrievedService = client.GetLinkedService("MyLinkedService"); #endregion #region Snippet:ListLinkedServices Pageable <LinkedServiceResource> linkedServices = client.GetLinkedServicesByWorkspace(); foreach (LinkedServiceResource linkedService in linkedServices) { System.Console.WriteLine(linkedService.Name); } #endregion #region Snippet:DeleteLinkedService client.StartDeleteLinkedService("MyLinkedService"); #endregion }
public LinkedServiceResource CreateOrUpdateLinkedService(string linkedServiceName, string rawJsonContent) { LinkedServiceResource linkedService = JsonConvert.DeserializeObject <LinkedServiceResource>(rawJsonContent); var operation = _linkedServiceClient.StartCreateOrUpdateLinkedService(linkedServiceName, linkedService); return(operation.Poll().Value); }
public void CreateLinkedService() { #region Snippet:CreateLinkedService var operation = LinkedServiceClient.StartCreateOrUpdateLinkedService("MyLinkedService", new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/"))); LinkedServiceResource linkedService = operation.WaitForCompletionAsync().ConfigureAwait(true).GetAwaiter().GetResult(); #endregion }
public static async ValueTask <LinkedServiceResource> CreateResource(LinkedServiceClient client, TestRecording recording) { string linkedServiceName = recording.GenerateId("LinkedService", 16); LinkedServiceResource resource = new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/")); LinkedServiceCreateOrUpdateLinkedServiceOperation operation = await client.StartCreateOrUpdateLinkedServiceAsync(linkedServiceName, resource); return(await operation.WaitForCompletionAsync()); }
public void LinkedServices_Update() { RunTest("LinkedServices_Update", (example, client, responseCode) => { LinkedServiceResource resource = client.LinkedServices.CreateOrUpdate(RGN(example), FN(example), LSN(example), LSR(example, client)); CheckResponseBody(example, client, responseCode, resource); }); }
public void LinkedServices_Get() { RunTest("LinkedServices_Get", (example, client, responseCode) => { LinkedServiceResource resource = client.LinkedServices.Get(RGN(example), FN(example), LSN(example)); CheckResponseBody(example, client, responseCode, resource); }); }
public async Task TestCreateLinkedService() { string linkedServiceName = Recording.GenerateName("LinkedSercive"); LinkedServiceCreateOrUpdateLinkedServiceOperation operation = await LinkedServiceClient.StartCreateOrUpdateLinkedServiceAsync(linkedServiceName, new LinkedServiceResource(new AzureDataLakeStoreLinkedService("adl://test.azuredatalakestore.net/"))); LinkedServiceResource linkedService = await operation.WaitForCompletionAsync(); Assert.AreEqual(linkedServiceName, linkedService.Name); }
internal CreateDataFlowDebugSessionRequest(string dataFlowName, string existingClusterId, int?clusterTimeout, string newClusterName, string newClusterNodeType, LinkedServiceResource dataBricksLinkedService) { DataFlowName = dataFlowName; ExistingClusterId = existingClusterId; ClusterTimeout = clusterTimeout; NewClusterName = newClusterName; NewClusterNodeType = newClusterNodeType; DataBricksLinkedService = dataBricksLinkedService; }
public PSLinkedServiceResource(LinkedServiceResource linkedServiceResource, string workspaceName) : base(linkedServiceResource?.Id, linkedServiceResource?.Name, linkedServiceResource?.Type, linkedServiceResource?.Etag) { this.WorkspaceName = workspaceName; this.Properties = new PSLinkedService(linkedServiceResource?.Properties); }
private static void createDataWarehouse(DataFactoryManagementClient client) { var IR = new IntegrationRuntimeReference(DatosGrales.azureIntegrationRuntime); var conStr = new SecureString(DatosGrales.warehouseConnectionString); LinkedServiceResource SqlServerLinkedServiceWarehouse = new LinkedServiceResource( new AzureSqlDWLinkedService(conStr, null, IR, "Sql Warehouse - " + DatosGrales.nombreBDWarehouse)); client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, "SqlServerLinkedService-Warehouse", SqlServerLinkedServiceWarehouse); }
public async Task TestDeleteLinkedService() { LinkedServiceClient client = CreateClient(); LinkedServiceResource resource = await DisposableLinkedService.CreateResource(client, this.Recording); LinkedServiceDeleteLinkedServiceOperation operation = await client.StartDeleteLinkedServiceAsync(resource.Name); await operation.WaitAndAssertSuccessfulCompletion(); }
public async Task TestGetLinkedService() { await foreach (var expectedLinkedService in LinkedServiceClient.GetLinkedServicesByWorkspaceAsync()) { LinkedServiceResource actualLinkedService = await LinkedServiceClient.GetLinkedServiceAsync(expectedLinkedService.Name); Assert.AreEqual(expectedLinkedService.Name, actualLinkedService.Name); Assert.AreEqual(expectedLinkedService.Id, actualLinkedService.Id); } }
private static void createAzureSSIS(DataFactoryManagementClient client) { var IR = new IntegrationRuntimeReference(DatosGrales.onPremiseIntegrationRuntime); var pass = new SecureString(DatosGrales.passwordWarehouse); LinkedServiceResource SqlServerLinkedServiceSSIS = new LinkedServiceResource( new SqlServerLinkedService( new SecureString(@"Data Source=sqlsrvbi00.database.windows.net;Initial Catalog=SSISDB;Integrated Security=False"), null, IR, "Sql que hostea el SSIS", DatosGrales.usuarioWarehouse, pass)); client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, DatosGrales.linkedServiceSSIS, SqlServerLinkedServiceSSIS); }
private static void createSQLServers(DataFactoryManagementClient client) { var IR = new IntegrationRuntimeReference(DatosGrales.onPremiseIntegrationRuntime); string nombreBD = DatosGrales.nombreBD; var pass = new SecureString(DatosGrales.passwordOnPremise); LinkedServiceResource SqlServerLinkedServiceClaim = new LinkedServiceResource( new SqlServerLinkedService( new SecureString(@"Data Source=ROW2K12SQL11;Initial Catalog=" + nombreBD + ";Integrated Security=True"), null, IR, "Sql Local - " + nombreBD, DatosGrales.usuarioOnPremise, pass)); client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, "SqlServerLinkedService-" + nombreBD, SqlServerLinkedServiceClaim); }
public PSLinkedService(LinkedServiceResource linkedService, string resourceGroupName, string factoryName) { if (linkedService == null) { throw new ArgumentNullException("linkedService"); } this.linkedService = linkedService; this.ResourceGroupName = resourceGroupName; this.DataFactoryName = factoryName; }
public virtual PSLinkedService GetLinkedService(string resourceGroupName, string dataFactoryName, string linkedServiceName) { LinkedServiceResource response = this.DataFactoryManagementClient.LinkedServices.Get(resourceGroupName, dataFactoryName, linkedServiceName); if (response == null) { return(null); } return(new PSLinkedService(response, resourceGroupName, dataFactoryName)); }
public LinkedServiceResource CreateOrUpdateLinkedService(string linkedServiceName, string rawJsonContent) { PSLinkedServiceResource psLinkedService = JsonConvert.DeserializeObject <PSLinkedServiceResource>(rawJsonContent, Settings); LinkedServiceResource linkedService = psLinkedService.ToSdkObject(); var operation = _linkedServiceClient.StartCreateOrUpdateLinkedService(linkedServiceName, linkedService); while (!operation.HasValue) { operation.UpdateStatus(); } return(operation.Value); }
internal static LinkedServiceResource GetLinkedServiceResource(string description) { LinkedServiceResource resource = new LinkedServiceResource { Properties = new AzureDataLakeStoreLinkedService() { Description = description, DataLakeStoreUri = "adl://test.azuredatalakestore.net/" } }; return(resource); }
private LinkedServiceResource GetLinkedServiceResource(string description) { LinkedServiceResource resource = new LinkedServiceResource { Properties = new AzureStorageLinkedService { Description = description, ConnectionString = new SecureString(string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", secrets.StorageAccountName, secrets.StorageAccountKey)) } }; return(resource); }
private static void createDataLake(DataFactoryManagementClient client) { AzureDataLakeStoreLinkedService DataLakeLinkedService = new AzureDataLakeStoreLinkedService(); DataLakeLinkedService.DataLakeStoreUri = "https://datalakebi00.azuredatalakestore.net/webhdfs/v1"; DataLakeLinkedService.ServicePrincipalId = DatosGrales.applicationId; DataLakeLinkedService.ServicePrincipalKey = new SecureString(DatosGrales.authenticationKey); DataLakeLinkedService.ResourceGroupName = DatosGrales.resourceGroup; DataLakeLinkedService.SubscriptionId = DatosGrales.subscriptionId; DataLakeLinkedService.Tenant = DatosGrales.tenantID; LinkedServiceResource DataLakeResource = new LinkedServiceResource(DataLakeLinkedService); DataLakeResource.Properties.ConnectVia = new IntegrationRuntimeReference(DatosGrales.azureIntegrationRuntime); client.LinkedServices.CreateOrUpdate(DatosGrales.resourceGroup, DatosGrales.dataFactoryName, DatosGrales.linkedServiceLake, DataLakeResource); }
public static void AddAzureSqlDatabaseLinkedService( this DataFactoryManagementClient client, AzureConfig config, string sqlDatabaseConnectionString, string sqlDatabaseLinkedServiceName) { Console.WriteLine($"Creating linked service {sqlDatabaseLinkedServiceName}..."); var storageLinkedService = new LinkedServiceResource( new AzureSqlDatabaseLinkedService { ConnectionString = sqlDatabaseConnectionString } ); client.LinkedServices.CreateOrUpdate(config.ResourceGroup, config.DataFactoryName, sqlDatabaseLinkedServiceName, storageLinkedService); Console.WriteLine(SafeJsonConvert.SerializeObject(storageLinkedService, client.SerializationSettings)); }
public async Task TestGetLinkedService() { LinkedServiceClient client = CreateClient(); await using DisposableLinkedService service = await DisposableLinkedService.Create(client, this.Recording); IList <LinkedServiceResource> services = await client.GetLinkedServicesByWorkspaceAsync().ToListAsync(); Assert.GreaterOrEqual(services.Count, 1); foreach (var expectedLinkedService in services) { LinkedServiceResource actualLinkedService = await client.GetLinkedServiceAsync(expectedLinkedService.Name); Assert.AreEqual(expectedLinkedService.Name, actualLinkedService.Name); Assert.AreEqual(expectedLinkedService.Id, actualLinkedService.Id); } }
public static void CreateBlobLS(DataFactoryManagementClient client) { Console.WriteLine("Creating linked service " + Constants.storageLinkedServiceName + "..."); LinkedServiceResource storageLinkedService = new LinkedServiceResource( new AzureStorageLinkedService { ConnectionString = new SecureString( "DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount + ";AccountKey=" + Constants.storageKey) } ); client.LinkedServices.CreateOrUpdate( Constants.resourceGroup, Constants.dataFactoryName, Constants.storageLinkedServiceName, storageLinkedService); //Console.WriteLine(SafeJsonConvert.SerializeObject( // storageLinkedService, client.SerializationSettings)); }
public async Task <ActionResult <LinkedServiceResource> > AddDataSource2Async(Guid engineId, string dataSourceName, [FromBody] LinkedService linkedService) { var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false); if (engine == null) { throw new Exception("Engine does not exists"); } var accessToken = await this.authProvider.GetAccessTokenForAppManagementAsync().ConfigureAwait(false); ServiceClientCredentials tokenCredentials = new TokenCredentials(accessToken); DataFactoryManagementClient client = new DataFactoryManagementClient(tokenCredentials) { SubscriptionId = this.options.SubscriptionId }; // Create the linked service resource LinkedServiceResource linkedServiceResource = new LinkedServiceResource(linkedService); string sensitiveString = linkedServiceResource.Properties switch { AzureStorageLinkedService azureStorageLinkedService => azureStorageLinkedService.ConnectionString.ToString(), _ => null }; if (!string.IsNullOrEmpty(sensitiveString)) { // Save the connection string to KeyVault await keyVaultsController.SetKeyVaultSecret(engineId, dataSourceName, new YKeyVaultSecretPayload { Key = dataSourceName, Value = sensitiveString }); } var newLinkedServiceResourceCreated = await client.LinkedServices.CreateOrUpdateAsync(engine.ResourceGroupName, engine.FactoryName, dataSourceName, linkedServiceResource); return(newLinkedServiceResourceCreated); }
public void LinkedService_WebLinkedServiceSDKSample() { string password = "******"; LinkedServiceResource linkedServiceResource = new LinkedServiceResource { Properties = new WebLinkedService { TypeProperties = new WebBasicAuthentication { Url = "https://localhost", Username = "******", Password = new SecureString(password) } } }; var handler = new RecordedDelegatingHandler(); var client = this.CreateWorkflowClient(handler); var json = SafeJsonConvert.SerializeObject(linkedServiceResource, client.SerializationSettings); Assert.Contains(password, json); }
public async Task TestRenameLinkedService() { LinkedServiceClient client = CreateClient(); LinkedServiceResource resource = await DisposableLinkedService.CreateResource(client, Recording); string newLinkedServiceName = Recording.GenerateId("LinkedService2", 16); LinkedServiceRenameLinkedServiceOperation renameOperation = await client.StartRenameLinkedServiceAsync(resource.Name, new ArtifactRenameRequest() { NewName = newLinkedServiceName }); await renameOperation.WaitForCompletionResponseAsync(); LinkedServiceResource service = await client.GetLinkedServiceAsync(newLinkedServiceName); Assert.AreEqual(newLinkedServiceName, service.Name); LinkedServiceDeleteLinkedServiceOperation operation = await client.StartDeleteLinkedServiceAsync(newLinkedServiceName); await operation.WaitForCompletionResponseAsync(); }
static void Main(string[] args) { // Set variables string tenantID = "<your tenant ID>"; string applicationId = "<your application ID>"; string authenticationKey = "<your authentication key for the application>"; string subscriptionId = "<your subscription ID where the data factory resides>"; string resourceGroup = "<your resource group where the data factory resides>"; string region = "<the location of your resource group>"; string dataFactoryName = "<specify the name of data factory to create. It must be globally unique.>"; string storageAccount = "<your storage account name to copy data>"; string storageKey = "<your storage account key>"; // specify the container and input folder from which all files // need to be copied to the output folder. string inputBlobPath = "<path to existing blob(s) to copy data from, e.g. containername/inputdir>"; //specify the contains and output folder where the files are copied string outputBlobPath = "<the blob path to copy data to, e.g. containername/outputdir>"; // name of the Azure Storage linked service, blob dataset, and the pipeline string storageLinkedServiceName = "AzureStorageLinkedService"; string blobDatasetName = "BlobDataset"; string pipelineName = "Adfv2QuickStartPipeline"; // Authenticate and create a data factory management client var context = new AuthenticationContext("https://login.microsoftonline.com/" + tenantID); ClientCredential cc = new ClientCredential(applicationId, authenticationKey); AuthenticationResult result = context.AcquireTokenAsync( "https://management.azure.com/", cc).Result; ServiceClientCredentials cred = new TokenCredentials(result.AccessToken); var client = new DataFactoryManagementClient(cred) { SubscriptionId = subscriptionId }; // Create a data factory Console.WriteLine("Creating data factory " + dataFactoryName + "..."); Factory dataFactory = new Factory { Location = region, Identity = new FactoryIdentity() }; client.Factories.CreateOrUpdate(resourceGroup, dataFactoryName, dataFactory); Console.WriteLine( SafeJsonConvert.SerializeObject(dataFactory, client.SerializationSettings)); while (client.Factories.Get(resourceGroup, dataFactoryName).ProvisioningState == "PendingCreation") { System.Threading.Thread.Sleep(1000); } // Create an Azure Storage linked service Console.WriteLine("Creating linked service " + storageLinkedServiceName + "..."); LinkedServiceResource storageLinkedService = new LinkedServiceResource( new AzureStorageLinkedService { ConnectionString = new SecureString( "DefaultEndpointsProtocol=https;AccountName=" + storageAccount + ";AccountKey=" + storageKey) } ); client.LinkedServices.CreateOrUpdate( resourceGroup, dataFactoryName, storageLinkedServiceName, storageLinkedService); Console.WriteLine(SafeJsonConvert.SerializeObject( storageLinkedService, client.SerializationSettings)); // Create an Azure Blob dataset Console.WriteLine("Creating dataset " + blobDatasetName + "..."); DatasetResource blobDataset = new DatasetResource( new AzureBlobDataset { LinkedServiceName = new LinkedServiceReference { ReferenceName = storageLinkedServiceName }, FolderPath = new Expression { Value = "@{dataset().path}" }, Parameters = new Dictionary <string, ParameterSpecification> { { "path", new ParameterSpecification { Type = ParameterType.String } } } } ); client.Datasets.CreateOrUpdate( resourceGroup, dataFactoryName, blobDatasetName, blobDataset); Console.WriteLine( SafeJsonConvert.SerializeObject(blobDataset, client.SerializationSettings)); // Create a pipeline with a copy activity Console.WriteLine("Creating pipeline " + pipelineName + "..."); PipelineResource pipeline = new PipelineResource { Parameters = new Dictionary <string, ParameterSpecification> { { "inputPath", new ParameterSpecification { Type = ParameterType.String } }, { "outputPath", new ParameterSpecification { Type = ParameterType.String } } }, Activities = new List <Activity> { new CopyActivity { Name = "CopyFromBlobToBlob", Inputs = new List <DatasetReference> { new DatasetReference() { ReferenceName = blobDatasetName, Parameters = new Dictionary <string, object> { { "path", "@pipeline().parameters.inputPath" } } } }, Outputs = new List <DatasetReference> { new DatasetReference { ReferenceName = blobDatasetName, Parameters = new Dictionary <string, object> { { "path", "@pipeline().parameters.outputPath" } } } }, Source = new BlobSource { }, Sink = new BlobSink { } } } }; client.Pipelines.CreateOrUpdate(resourceGroup, dataFactoryName, pipelineName, pipeline); Console.WriteLine(SafeJsonConvert.SerializeObject(pipeline, client.SerializationSettings)); // Create a pipeline run Console.WriteLine("Creating pipeline run..."); Dictionary <string, object> parameters = new Dictionary <string, object> { { "inputPath", inputBlobPath }, { "outputPath", outputBlobPath } }; CreateRunResponse runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync( resourceGroup, dataFactoryName, pipelineName, parameters: parameters ).Result.Body; Console.WriteLine("Pipeline run ID: " + runResponse.RunId); // Monitor the pipeline run Console.WriteLine("Checking pipeline run status..."); PipelineRun pipelineRun; while (true) { pipelineRun = client.PipelineRuns.Get( resourceGroup, dataFactoryName, runResponse.RunId); Console.WriteLine("Status: " + pipelineRun.Status); if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued") { System.Threading.Thread.Sleep(15000); } else { break; } } // Check the copy activity run details Console.WriteLine("Checking copy activity run details..."); RunFilterParameters filterParams = new RunFilterParameters( DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10)); ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun( resourceGroup, dataFactoryName, runResponse.RunId, filterParams); if (pipelineRun.Status == "Succeeded") { Console.WriteLine(queryResponse.Value.First().Output); } else { Console.WriteLine(queryResponse.Value.First().Error); } Console.WriteLine("\nPress any key to exit..."); Console.ReadKey(); }
public PSLinkedService() { linkedService = new LinkedServiceResource(); }