public static async ValueTask <PipelineResource> CreateResource(PipelineClient client, TestRecording recording)
{
string pipelineName = recording.GenerateId("Pipeline", 16);
PipelineCreateOrUpdatePipelineOperation createOperation = await client.StartCreateOrUpdatePipelineAsync(pipelineName, new PipelineResource());
return(await createOperation.WaitForCompletionAsync());
}
public SynapseAnalyticsArtifactsClient(string workspaceName, IAzureContext context)
{
if (context == null)
{
throw new SynapseException(Resources.InvalidDefaultSubscription);
}
Settings = new JsonSerializerSettings
{
DateFormatHandling = Newtonsoft.Json.DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = Newtonsoft.Json.DateTimeZoneHandling.Utc,
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Serialize,
ContractResolver = new ReadOnlyJsonContractResolver(),
Converters = new List <JsonConverter>
{
new Iso8601TimeSpanConverter()
}
};
Settings.Converters.Add(new TransformationJsonConverter());
Settings.Converters.Add(new PolymorphicDeserializeJsonConverter <PSActivity>("type"));
Settings.Converters.Add(new PolymorphicDeserializeJsonConverter <PSLinkedService>("type"));
Settings.Converters.Add(new PolymorphicDeserializeJsonConverter <PSTrigger>("type"));
string suffix = context.Environment.GetEndpoint(AzureEnvironment.ExtendedEndpoint.AzureSynapseAnalyticsEndpointSuffix);
Uri uri = new Uri("https://" + workspaceName + "." + suffix);
_pipelineClient = new PipelineClient(uri, new AzureSessionCredential(context));
_pipelineRunClient = new PipelineRunClient(uri, new AzureSessionCredential(context));
_linkedServiceClient = new LinkedServiceClient(uri, new AzureSessionCredential(context));
_notebookClient = new NotebookClient(uri, new AzureSessionCredential(context));
_triggerClient = new TriggerClient(uri, new AzureSessionCredential(context));
_triggerRunClient = new TriggerRunClient(uri, new AzureSessionCredential(context));
}
public async Task CreateAndRunPipeline()
{
const string PipelineName = "Test-Pipeline";
const string JobName = "SparkJobName";
const string ActivityName = "ActivityName";
string workspaceUrl = TestEnvironment.WorkspaceUrl;
var pipelineClient = new PipelineClient(endpoint: new Uri(workspaceUrl), credential: new DefaultAzureCredential());
var sparkJob = new SynapseSparkJobReference(SparkJobReferenceType.SparkJobDefinitionReference, JobName);
var activity = new SynapseSparkJobDefinitionActivity(ActivityName, sparkJob);
var pipelineResource = new PipelineResource();
pipelineResource.Activities.Add(activity);
Console.WriteLine("Create pipeline if not already exists.");
await pipelineClient.StartCreateOrUpdatePipelineAsync(PipelineName, pipelineResource);
Console.WriteLine("Pipeline created");
Console.WriteLine("Running pipeline.");
CreateRunResponse runOperation = await pipelineClient.CreatePipelineRunAsync(PipelineName);
Console.WriteLine("Run started. ID: {0}", runOperation.RunId);
}
public async Task PipelineSample()
{
#region Snippet:CreatePipelineClient
// Replace the string below with your actual endpoint url.
string endpoint = "<my-endpoint-url>";
/*@@*/ endpoint = TestEnvironment.EndpointUrl;
PipelineClient client = new PipelineClient(endpoint: new Uri(endpoint), credential: new DefaultAzureCredential());
#endregion
#region Snippet:CreatePipeline
PipelineCreateOrUpdatePipelineOperation operation = client.StartCreateOrUpdatePipeline("MyPipeline", new PipelineResource());
Response <PipelineResource> createdPipeline = await operation.WaitForCompletionAsync();
#endregion
#region Snippet:RetrievePipeline
PipelineResource retrievedPipeline = client.GetPipeline("MyPipeline");
#endregion
#region Snippet:ListPipelines
Pageable <PipelineResource> pipelines = client.GetPipelinesByWorkspace();
foreach (PipelineResource pipeline in pipelines)
{
System.Console.WriteLine(pipeline.Name);
}
#endregion
#region Snippet:DeletePipeline
client.StartDeletePipeline("MyPipeline");
#endregion
}
public SynapseAnalyticsArtifactsClient(string workspaceName, IAzureContext context)
{
if (context == null)
{
throw new AzPSInvalidOperationException(Resources.InvalidDefaultSubscription);
}
_context = context;
string suffix = context.Environment.GetEndpoint(AzureEnvironment.ExtendedEndpoint.AzureSynapseAnalyticsEndpointSuffix);
Uri uri = new Uri("https://" + workspaceName + "." + suffix);
_endpoint = uri;
_pipelineClient = new PipelineClient(uri, new AzureSessionCredential(context));
_pipelineRunClient = new PipelineRunClient(uri, new AzureSessionCredential(context));
_linkedServiceClient = new LinkedServiceClient(uri, new AzureSessionCredential(context));
_notebookClient = new NotebookClient(uri, new AzureSessionCredential(context));
_triggerClient = new TriggerClient(uri, new AzureSessionCredential(context));
_triggerRunClient = new TriggerRunClient(uri, new AzureSessionCredential(context));
_datasetClient = new DatasetClient(uri, new AzureSessionCredential(context));
_dataFlowClient = new DataFlowClient(uri, new AzureSessionCredential(context));
_dataFlowDebugSessionClient = new DataFlowDebugSessionClient(uri, new AzureSessionCredential(context));
_bigDataPoolsClient = new BigDataPoolsClient(uri, new AzureSessionCredential(context));
_sparkJobDefinitionClient = new SparkJobDefinitionClient(uri, new AzureSessionCredential(context));
_sqlScriptClient = new SqlScriptClient(uri, new AzureSessionCredential(context));
}
public AzureSynapseService(PipelineRequest request, ILogger logger)
{
_logger = logger;
_logger.LogInformation("Creating SYN connectivity clients.");
//Auth details
var context = new AuthenticationContext("https://login.windows.net/" + request.TenantId);
var cc = new ClientCredential(request.ApplicationId, request.AuthenticationKey);
var result = context.AcquireTokenAsync("https://management.azure.com/", cc).Result;
var cred = new TokenCredentials(result.AccessToken);
//Management Client
_synManagementClient = new SynapseManagementClient(cred)
{
SubscriptionId = request.SubscriptionId
};
//Pipeline Clients
Uri synapseDevEndpoint = new Uri("https://" + request.OrchestratorName.ToLower() + ".dev.azuresynapse.net");
TokenCredential token = new ClientSecretCredential
(
request.TenantId,
request.ApplicationId,
request.AuthenticationKey
);
_pipelineClient = new PipelineClient(synapseDevEndpoint, token);
_pipelineRunClient = new PipelineRunClient(synapseDevEndpoint, token);
}
private void InstantiatePipelineClient(RelayNodeGroupDefinition groupDefinition)
{
var client = PipelineClient.GetClient(_ipEndPoint);
client.AckTimeout = TimeSpan.FromMilliseconds(groupDefinition.SocketSettings.ReceiveTimeout >> 1);
client.ResponseTimeout = TimeSpan.FromMilliseconds(groupDefinition.SocketSettings.ReceiveTimeout);
RelayMessageSender = new PipelineSender(client);
}
public async Task TestCreatePipeline()
{
string pipelineName = Recording.GenerateName("Pipeline");
PipelineCreateOrUpdatePipelineOperation operation = await PipelineClient.StartCreateOrUpdatePipelineAsync(pipelineName, new PipelineResource());
PipelineResource pipeline = await operation.WaitForCompletionAsync();
Assert.AreEqual(pipelineName, pipeline.Name);
}
public async Task TestPipelineRun()
{
PipelineClient client = CreateClient();
await using DisposablePipeline pipeline = await DisposablePipeline.Create(client, this.Recording);
CreateRunResponse runResponse = await client.CreatePipelineRunAsync(pipeline.Name);
Assert.NotNull(runResponse.RunId);
}
public async Task TestDeletePipeline()
{
var operation = await PipelineClient.StartDeletePipelineAsync("MyPipeline");
while (!operation.HasValue)
{
operation.UpdateStatus();
}
Assert.AreEqual(200, operation.Value.Status);
}
public async Task TestGetPipeline()
{
await foreach (var expectedPipeline in PipelineClient.GetPipelinesByWorkspaceAsync())
{
PipelineResource actualPipeline = await PipelineClient.GetPipelineAsync(expectedPipeline.Name);
Assert.AreEqual(expectedPipeline.Name, actualPipeline.Name);
Assert.AreEqual(expectedPipeline.Id, actualPipeline.Id);
}
}
public async Task TestCreatePipeline()
{
var operation = await PipelineClient.StartCreateOrUpdatePipelineAsync("MyPipeline", new PipelineResource());
while (!operation.HasValue)
{
operation.UpdateStatus();
}
Assert.AreEqual("MyPipeline", operation.Value.Name);
}
public async Task TestDeleteNotebook()
{
PipelineClient client = CreateClient();
PipelineResource resource = await DisposablePipeline.CreateResource(client, this.Recording);
PipelineDeletePipelineOperation operation = await client.StartDeletePipelineAsync(resource.Name);
await operation.WaitAndAssertSuccessfulCompletion();
}
private async Task RunPipeline(string pipelineName)
{
string endpoint = TestEnvironment.EndpointUrl;
var pipelineClient = new PipelineClient(endpoint: new Uri(endpoint), credential: new DefaultAzureCredential());
Console.WriteLine("Running pipeline.");
CreateRunResponse runOperation = await pipelineClient.CreatePipelineRunAsync(pipelineName);
Console.WriteLine("Run started. ID: {0}", runOperation.RunId);
}
public async Task RunPipeline()
{
const string PipelineName = "Test-Pipeline";
string workspaceUrl = TestEnvironment.WorkspaceUrl;
var pipelineClient = new PipelineClient(endpoint: new Uri(workspaceUrl), credential: new DefaultAzureCredential());
Console.WriteLine("Running pipeline.");
CreateRunResponse runOperation = await pipelineClient.CreatePipelineRunAsync(PipelineName);
Console.WriteLine("Run started. ID: {0}", runOperation.RunId);
}
public void CreateClient()
{
// Environment variable with the Synapse workspace endpoint.
string workspaceUrl = TestEnvironment.WorkspaceUrl;
#region Snippet:CreatePipelineClient
// Create a new Pipeline client using the default credential from Azure.Identity using environment variables previously set,
// including AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, and AZURE_TENANT_ID.
PipelineClient client = new PipelineClient(endpoint: new Uri(workspaceUrl), credential: new DefaultAzureCredential());
#endregion
this.PipelineClient = client;
}
public async Task TestDeletePipeline()
{
string pipelineName = Recording.GenerateName("Pipeline");
PipelineCreateOrUpdatePipelineOperation createOperation = await PipelineClient.StartCreateOrUpdatePipelineAsync(pipelineName, new PipelineResource());
await createOperation.WaitForCompletionAsync();
PipelineDeletePipelineOperation deleteOperation = await PipelineClient.StartDeletePipelineAsync(pipelineName);
Response response = await deleteOperation.WaitForCompletionAsync();
Assert.AreEqual(200, response.Status);
}
public async Task RunPipeline()
{
#region Snippet:CreatePipelineClientPrep
#if SNIPPET
// Replace the string below with your actual endpoint url.
string endpoint = "<my-endpoint-url>";
#else
string endpoint = TestEnvironment.EndpointUrl;
#endif
string pipelineName = "Test-Pipeline";
#endregion
#region Snippet:CreatePipelineClient
var client = new PipelineClient(endpoint: new Uri(endpoint), credential: new DefaultAzureCredential());
#endregion
#region Snippet:CreatePipeline
PipelineCreateOrUpdatePipelineOperation operation = client.StartCreateOrUpdatePipeline(pipelineName, new PipelineResource());
Response <PipelineResource> createdPipeline = await operation.WaitForCompletionAsync();
#endregion
#region Snippet:RetrievePipeline
PipelineResource retrievedPipeline = client.GetPipeline(pipelineName);
Console.WriteLine("Pipeline ID: {0}", retrievedPipeline.Id);
#endregion
#region Snippet:RunPipeline
Console.WriteLine("Running pipeline.");
CreateRunResponse runOperation = await client.CreatePipelineRunAsync(pipelineName);
Console.WriteLine("Run started. ID: {0}", runOperation.RunId);
#endregion
#region Snippet:ListPipelines
Pageable <PipelineResource> pipelines = client.GetPipelinesByWorkspace();
foreach (PipelineResource pipeline in pipelines)
{
Console.WriteLine(pipeline.Name);
}
#endregion
#region Snippet:DeletePipeline
PipelineDeletePipelineOperation deleteOperation = client.StartDeletePipeline(pipelineName);
await deleteOperation.WaitForCompletionResponseAsync();
#endregion
}
public async Task TestQueryRuns()
{
PipelineClient pipelineClient = CreatePipelineClient();
PipelineRunClient runClient = CreatePipelineRunClient();
await using DisposablePipeline pipeline = await DisposablePipeline.Create(pipelineClient, this.Recording);
CreateRunResponse runResponse = await pipelineClient.CreatePipelineRunAsync(pipeline.Name);
Assert.NotNull(runResponse.RunId);
PipelineRunsQueryResponse queryResponse = await runClient.QueryPipelineRunsByWorkspaceAsync(new RunFilterParameters (DateTimeOffset.MinValue, DateTimeOffset.MaxValue));
Assert.GreaterOrEqual(queryResponse.Value.Count, 1);
}
public async Task TestCancelRun()
{
PipelineClient pipelineClient = CreatePipelineClient();
PipelineRunClient runClient = CreatePipelineRunClient();
await using DisposablePipeline pipeline = await DisposablePipeline.Create(pipelineClient, this.Recording);
CreateRunResponse runResponse = await pipelineClient.CreatePipelineRunAsync(pipeline.Name);
Assert.NotNull(runResponse.RunId);
Response response = await runClient.CancelPipelineRunAsync(runResponse.RunId);
response.AssertSuccess();
}
public async Task TestGet()
{
PipelineClient pipelineClient = CreatePipelineClient();
PipelineRunClient runClient = CreatePipelineRunClient();
await using DisposablePipeline pipeline = await DisposablePipeline.Create(pipelineClient, this.Recording);
CreateRunResponse runResponse = await pipelineClient.CreatePipelineRunAsync(pipeline.Name);
Assert.NotNull(runResponse.RunId);
PipelineRun run = await runClient.GetPipelineRunAsync(runResponse.RunId);
Assert.AreEqual(run.RunId, runResponse.RunId);
Assert.NotNull(run.Status);
}
public async Task TestGetPipeline()
{
PipelineClient client = CreateClient();
await using DisposablePipeline pipeline = await DisposablePipeline.Create(client, this.Recording);
IList <PipelineResource> pipelines = await client.GetPipelinesByWorkspaceAsync().ToListAsync();
Assert.GreaterOrEqual(pipelines.Count, 1);
foreach (var expectedPipeline in pipelines)
{
PipelineResource actualPipeline = await client.GetPipelineAsync(expectedPipeline.Name);
Assert.AreEqual(expectedPipeline.Name, actualPipeline.Name);
Assert.AreEqual(expectedPipeline.Id, actualPipeline.Id);
}
}
private async Task CreatePipeline(string pipelineName)
{
const string JobName = "SparkJobName";
const string ActivityName = "ActivityName";
string endpoint = TestEnvironment.EndpointUrl;
var pipelineClient = new PipelineClient(endpoint: new Uri(endpoint), credential: new DefaultAzureCredential());
var sparkJob = new SynapseSparkJobReference(SparkJobReferenceType.SparkJobDefinitionReference, JobName);
var activity = new SynapseSparkJobDefinitionActivity(ActivityName, sparkJob);
var pipelineResource = new PipelineResource();
pipelineResource.Activities.Add(activity);
Console.WriteLine("Create pipeline if not already exists.");
PipelineCreateOrUpdatePipelineOperation operation = await pipelineClient.StartCreateOrUpdatePipelineAsync(pipelineName, pipelineResource);
await operation.WaitForCompletionAsync();
Console.WriteLine("Pipeline created");
}
public async Task TestRenameLinkedService()
{
PipelineClient client = CreateClient();
PipelineResource resource = await DisposablePipeline.CreateResource(client, Recording);
string newPipelineName = Recording.GenerateId("Pipeline2", 16);
PipelineRenamePipelineOperation renameOperation = await client.StartRenamePipelineAsync(resource.Name, new ArtifactRenameRequest()
{
NewName = newPipelineName
});
await renameOperation.WaitForCompletionResponseAsync();
PipelineResource pipeline = await client.GetPipelineAsync(newPipelineName);
Assert.AreEqual(newPipelineName, pipeline.Name);
PipelineDeletePipelineOperation operation = await client.StartDeletePipelineAsync(newPipelineName);
await operation.WaitForCompletionResponseAsync();
}
public async Task TestStartStop()
{
TriggerClient client = CreateClient();
PipelineClient pipelineClient = CreatePipelineClient();
await using DisposableTrigger trigger = await DisposableTrigger.Create(client, Recording);
await using DisposablePipeline pipeline = await DisposablePipeline.Create(pipelineClient, Recording);
// SYNAPSE_API_ISSUE - How do we point the trigger to our pipeline
TriggerStartTriggerOperation startOperation = await client.StartStartTriggerAsync(trigger.Name);
Response startResponse = await startOperation.WaitForCompletionResponseAsync();
startResponse.AssertSuccess();
TriggerStopTriggerOperation stopOperation = await client.StartStopTriggerAsync(trigger.Name);
Response stopResponse = await stopOperation.WaitForCompletionResponseAsync();
stopResponse.AssertSuccess();
}
public static async ValueTask <DisposablePipeline> Create(PipelineClient client, TestRecording recording) => new DisposablePipeline(client, await CreateResource(client, recording));
public PipelineSender(PipelineClient client)
{
_client = client;
}
private DisposablePipeline(PipelineClient client, PipelineResource resource)
{
_client = client;
Resource = resource;
}
