public async Task TestCreateDataFlow()
{
string dataFlowName = Recording.GenerateName("DataFlow");
DataFlowCreateOrUpdateDataFlowOperation operation = await DataFlowClient.StartCreateOrUpdateDataFlowAsync(dataFlowName, new DataFlowResource(new DataFlow()));
DataFlowResource dataFlow = await operation.WaitForCompletionAsync();
Assert.AreEqual(dataFlowName, dataFlow.Name);
}
public PSDataFlowResource(DataFlowResource dataFlowResource, string workspaceName)
: base(dataFlowResource?.Id,
dataFlowResource?.Name,
dataFlowResource?.Type,
dataFlowResource?.Etag)
{
this.WorkspaceName = workspaceName;
this.Properties = dataFlowResource?.Properties;
}
public async Task GetDataFlow()
{
DataFlowClient client = CreateClient();
await using DisposableDataFlow flow = await DisposableDataFlow.Create(client, this.Recording);
DataFlowResource dataFlow = await client.GetDataFlowAsync(flow.Name);
Assert.AreEqual(flow.Name, dataFlow.Name);
}
public async Task TestGetDataFlow()
{
await foreach (var expectedDataFlow in DataFlowClient.GetDataFlowsByWorkspaceAsync())
{
DataFlowResource actualDataFlow = await DataFlowClient.GetDataFlowAsync(expectedDataFlow.Name);
Assert.AreEqual(expectedDataFlow.Name, actualDataFlow.Name);
Assert.AreEqual(expectedDataFlow.Id, actualDataFlow.Id);
}
}
public async Task DeleteDataFlow()
{
DataFlowClient client = CreateClient();
DataFlowResource resource = await DisposableDataFlow.CreateResource(client, this.Recording);
DataFlowDeleteDataFlowOperation operation = await client.StartDeleteDataFlowAsync(resource.Name);
await operation.WaitAndAssertSuccessfulCompletion();
}
public DataFlowResource CreateOrUpdateDataFlow(string dataFlowName, string rawJsonContent)
{
PSDataFlowResource pSDatasetResource = JsonConvert.DeserializeObject <PSDataFlowResource>(rawJsonContent, Settings);
DataFlowResource dataFlow = pSDatasetResource.ToSdkObject();
var operation = _dataFlowClient.StartCreateOrUpdateDataFlow(dataFlowName, dataFlow);
while (!operation.HasValue)
{
operation.UpdateStatus();
}
return(operation.Value);
}
internal static DataFlowResource GetDataFlowResource(string description, string datasetName)
{
DataFlowResource resource = new DataFlowResource
{
Properties = new MappingDataFlow
{
Description = description,
Sources = new List <DataFlowSource>()
{
new DataFlowSource()
{
Name = "source",
Description = "source 1",
Dataset = new DatasetReference()
{
ReferenceName = datasetName,
Parameters = new Dictionary <string, object>
{
{ "JobId", new Expression("@pipeline().parameters.JobId") }
}
}
}
},
Sinks = new List <DataFlowSink>()
{
new DataFlowSink()
{
Name = "sink",
Description = "sink 1",
Dataset = new DatasetReference()
{
ReferenceName = datasetName,
Parameters = new Dictionary <string, object>
{
{ "JobId", new Expression("@pipeline().parameters.JobId") }
}
}
}
},
Transformations = new List <Transformation>()
{
new Transformation()
{
Name = "transformation",
Description = "transformation 1"
}
}
}
};
return(resource);
}
public async Task DataFlowCrud()
{
string dataFlowName = "TestDataFlow";
string powerQueryName = "TestPowerQuery";
string datasetName = "TestDataFlowDataset";
string linkedServiceName = "TestDataFlowLinkedService";
Func <DataFactoryManagementClient, Task> action = async(client) =>
{
await DataFactoryScenarioTests.Create(client, this.ResourceGroupName, this.DataFactoryName, new Factory(location : FactoryLocation));
var expectedLinkedService = LinkedServiceScenarioTests.GetLinkedServiceResource(null);
await LinkedServiceScenarioTests.Create(client, this.ResourceGroupName, this.DataFactoryName, linkedServiceName, expectedLinkedService);
var expectedDataset = DatasetScenarioTests.GetDatasetResource(null, linkedServiceName);
await DatasetScenarioTests.Create(client, this.ResourceGroupName, this.DataFactoryName, datasetName, expectedDataset);
DataFlowResource expectedDataFlow = GetDataFlowResource(null, datasetName);
await Create(client, this.ResourceGroupName, this.DataFactoryName, dataFlowName, expectedDataFlow);
await GetList(client, this.ResourceGroupName, this.DataFactoryName, dataFlowName, expectedDataFlow);
DataFlowResource updatedDataFlow = GetDataFlowResource("data flow description", datasetName);
await Update(client, this.ResourceGroupName, this.DataFactoryName, dataFlowName, updatedDataFlow);
await GetList(client, this.ResourceGroupName, this.DataFactoryName, dataFlowName, updatedDataFlow);
await Delete(client, this.ResourceGroupName, this.DataFactoryName, dataFlowName);
// Power Query (Wrangling data flow)
DataFlowResource expectedPowerQuery = GetPowerQueryResource(null, datasetName);
await Create(client, this.ResourceGroupName, this.DataFactoryName, powerQueryName, expectedPowerQuery);
await GetList(client, this.ResourceGroupName, this.DataFactoryName, powerQueryName, expectedPowerQuery);
DataFlowResource updatedPowerQuery = GetPowerQueryResource("power query description", datasetName);
await Update(client, this.ResourceGroupName, this.DataFactoryName, powerQueryName, updatedPowerQuery);
await GetList(client, this.ResourceGroupName, this.DataFactoryName, powerQueryName, updatedPowerQuery);
await Delete(client, this.ResourceGroupName, this.DataFactoryName, powerQueryName);
await DatasetScenarioTests.Delete(client, this.ResourceGroupName, this.DataFactoryName, datasetName);
await LinkedServiceScenarioTests.Delete(client, this.ResourceGroupName, this.DataFactoryName, linkedServiceName);
};
Func <DataFactoryManagementClient, Task> finallyAction = async(client) =>
{
await client.Factories.DeleteAsync(this.ResourceGroupName, this.DataFactoryName);
};
await this.RunTest(action, finallyAction);
}
public PSDataFlow(DataFlowResource dataFlow, string resourceGroupName, string factoryName)
{
if (dataFlow == null)
{
throw new ArgumentNullException("dataFlow");
}
if (dataFlow.Properties == null)
{
dataFlow.Properties = new DataFlow();
}
this._dataFlow = dataFlow;
this.ResourceGroupName = resourceGroupName;
this.DataFactoryName = factoryName;
}
public async Task <Response <DataFlowResource> > GetDataFlowAsync(string dataFlowName, string ifNoneMatch = null, CancellationToken cancellationToken = default)
{
if (dataFlowName == null)
{
throw new ArgumentNullException(nameof(dataFlowName));
}
using var message = CreateGetDataFlowRequest(dataFlowName, ifNoneMatch);
await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false);
switch (message.Response.Status)
{
case 200:
{
DataFlowResource value = default;
using var document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false);
value = DataFlowResource.DeserializeDataFlowResource(document.RootElement);
return(Response.FromValue(value, message.Response));
}
public async Task DeleteDataFlow()
{
DataFlowClient client = CreateClient();
DataFlowResource resource = await DisposableDataFlow.CreateResource(client, this.Recording);
DataFlowDeleteDataFlowOperation operation = await client.StartDeleteDataFlowAsync(resource.Name);
Response response = await operation.WaitForCompletionAsync();
switch (response.Status)
{
case 200:
case 204:
break;
default:
Assert.Fail($"Unexpected status ${response.Status} returned");
break;
}
}
public async Task DataFlowSample()
{
#region Snippet:CreateDataFlowClientPrep
#if SNIPPET
// Replace the string below with your actual endpoint url.
string endpoint = "<my-endpoint-url>";
#else
string endpoint = TestEnvironment.EndpointUrl;
#endif
string dataFlowName = "Test-DataFlow";
#endregion
#region Snippet:CreateDataFlowClient
DataFlowClient client = new DataFlowClient(endpoint: new Uri(endpoint), credential: new DefaultAzureCredential());
#endregion
#region Snippet:CreateDataFlow
DataFlowCreateOrUpdateDataFlowOperation operation = client.StartCreateOrUpdateDataFlow(dataFlowName, new DataFlowResource(new DataFlow()));
Response <DataFlowResource> createdDataflow = await operation.WaitForCompletionAsync();
#endregion
#region Snippet:RetrieveDataFlow
DataFlowResource retrievedDataflow = client.GetDataFlow(dataFlowName);
#endregion
#region Snippet:ListDataFlows
Pageable <DataFlowResource> dataFlows = client.GetDataFlowsByWorkspace();
foreach (DataFlowResource dataflow in dataFlows)
{
System.Console.WriteLine(dataflow.Name);
}
#endregion
#region Snippet:DeleteDataFlow
DataFlowDeleteDataFlowOperation deleteOperation = client.StartDeleteDataFlow(dataFlowName);
await deleteOperation.WaitForCompletionResponseAsync();
#endregion
}
public Response CreateOrUpdateDataFlow(string dataFlowName, DataFlowResource dataFlow, string ifMatch = null, CancellationToken cancellationToken = default)
{
if (dataFlowName == null)
{
throw new ArgumentNullException(nameof(dataFlowName));
}
if (dataFlow == null)
{
throw new ArgumentNullException(nameof(dataFlow));
}
using var message = CreateCreateOrUpdateDataFlowRequest(dataFlowName, dataFlow, ifMatch);
_pipeline.Send(message, cancellationToken);
switch (message.Response.Status)
{
case 200:
case 202:
return(message.Response);
default:
throw _clientDiagnostics.CreateRequestFailedException(message.Response);
}
}
public async Task RenameDataFlow()
{
DataFlowClient client = CreateClient();
DataFlowResource resource = await DisposableDataFlow.CreateResource(client, this.Recording);
string newFlowName = Recording.GenerateAssetName("DataFlow2");
DataFlowRenameDataFlowOperation renameOperation = await client.StartRenameDataFlowAsync(resource.Name, new ArtifactRenameRequest()
{
NewName = newFlowName
});
await renameOperation.WaitForCompletionResponseAsync();
DataFlowResource dataFlow = await client.GetDataFlowAsync(newFlowName);
Assert.AreEqual(newFlowName, dataFlow.Name);
DataFlowDeleteDataFlowOperation operation = await client.StartDeleteDataFlowAsync(newFlowName);
await operation.WaitForCompletionResponseAsync();
}
public virtual DataFlowCreateOrUpdateDataFlowOperation StartCreateOrUpdateDataFlow(string dataFlowName, DataFlowResource dataFlow, string ifMatch = null, CancellationToken cancellationToken = default)
{
if (dataFlowName == null)
{
throw new ArgumentNullException(nameof(dataFlowName));
}
if (dataFlow == null)
{
throw new ArgumentNullException(nameof(dataFlow));
}
using var scope = _clientDiagnostics.CreateScope("DataFlowClient.StartCreateOrUpdateDataFlow");
scope.Start();
try
{
var originalResponse = RestClient.CreateOrUpdateDataFlow(dataFlowName, dataFlow, ifMatch, cancellationToken);
return(new DataFlowCreateOrUpdateDataFlowOperation(_clientDiagnostics, _pipeline, RestClient.CreateCreateOrUpdateDataFlowRequest(dataFlowName, dataFlow, ifMatch).Request, originalResponse));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
public void RetrieveDataFlow()
{
#region Snippet:RetrieveDataFlow
DataFlowResource dataFlow = DataFlowClient.GetDataFlow("MyDataFlow");
#endregion
}
public virtual Response <DataFlowResource> CreateOrUpdateDataFlow(string dataFlowName, DataFlowResource dataFlow, string ifMatch = null, CancellationToken cancellationToken = default)
{
using var scope = _clientDiagnostics.CreateScope("DataFlowClient.CreateOrUpdateDataFlow");
scope.Start();
try
{
return(RestClient.CreateOrUpdateDataFlow(dataFlowName, dataFlow, ifMatch, cancellationToken));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
private static void ValidateDataFlow(DataFactoryManagementClient client, string resourceGroupName, string dataFactoryName, DataFlowResource expected, DataFlowResource actual, string expectedName)
{
ValidateSubResource(client, resourceGroupName, actual, dataFactoryName, expectedName, "dataflows");
Assert.IsType <MappingDataFlow>(actual.Properties);
}
internal static async Task GetList(DataFactoryManagementClient client, string resourceGroupName, string dataFactoryName, string dataFlowName, DataFlowResource expectedDataFlow)
{
AzureOperationResponse <DataFlowResource> getDataFlowResponse = await client.DataFlows.GetWithHttpMessagesAsync(resourceGroupName, dataFactoryName, dataFlowName);
ValidateDataFlow(client, resourceGroupName, dataFactoryName, expectedDataFlow, getDataFlowResponse.Body, dataFlowName);
Assert.Equal(HttpStatusCode.OK, getDataFlowResponse.Response.StatusCode);
AzureOperationResponse <IPage <DataFlowResource> > listDataFlowResponse = await client.DataFlows.ListByFactoryWithHttpMessagesAsync(resourceGroupName, dataFactoryName);
ValidateDataFlow(client, resourceGroupName, dataFactoryName, expectedDataFlow, listDataFlowResponse.Body.First(), dataFlowName);
Assert.Equal(HttpStatusCode.OK, listDataFlowResponse.Response.StatusCode);
}
internal static async Task Update(DataFactoryManagementClient client, string resourceGroupName, string dataFactoryName, string dataFlowName, DataFlowResource expectedDataFlow)
{
AzureOperationResponse <DataFlowResource> updateDataFlowResponse = await client.DataFlows.CreateOrUpdateWithHttpMessagesAsync(resourceGroupName, dataFactoryName, dataFlowName, expectedDataFlow);
ValidateDataFlow(client, resourceGroupName, dataFactoryName, expectedDataFlow, updateDataFlowResponse.Body, dataFlowName);
Assert.Equal(HttpStatusCode.OK, updateDataFlowResponse.Response.StatusCode);
}
/// <summary>
/// Creates or updates a data flow.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The resource group name.
/// </param>
/// <param name='factoryName'>
/// The factory name.
/// </param>
/// <param name='dataFlowName'>
/// The data flow name.
/// </param>
/// <param name='dataFlow'>
/// Data flow resource definition.
/// </param>
/// <param name='ifMatch'>
/// ETag of the data flow entity. Should only be specified for update, for
/// which it should match existing entity or can be * for unconditional update.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static async Task <DataFlowResource> CreateOrUpdateAsync(this IDataFlowsOperations operations, string resourceGroupName, string factoryName, string dataFlowName, DataFlowResource dataFlow, string ifMatch = default(string), CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.CreateOrUpdateWithHttpMessagesAsync(resourceGroupName, factoryName, dataFlowName, dataFlow, ifMatch, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Creates or updates a data flow.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The resource group name.
/// </param>
/// <param name='factoryName'>
/// The factory name.
/// </param>
/// <param name='dataFlowName'>
/// The data flow name.
/// </param>
/// <param name='dataFlow'>
/// Data flow resource definition.
/// </param>
/// <param name='ifMatch'>
/// ETag of the data flow entity. Should only be specified for update, for
/// which it should match existing entity or can be * for unconditional update.
/// </param>
public static DataFlowResource CreateOrUpdate(this IDataFlowsOperations operations, string resourceGroupName, string factoryName, string dataFlowName, DataFlowResource dataFlow, string ifMatch = default(string))
{
return(operations.CreateOrUpdateAsync(resourceGroupName, factoryName, dataFlowName, dataFlow, ifMatch).GetAwaiter().GetResult());
}
public async Task <Response> CreateOrUpdateDataFlowAsync(string dataFlowName, DataFlowResource dataFlow, string ifMatch = null, CancellationToken cancellationToken = default)
{
if (dataFlowName == null)
{
throw new ArgumentNullException(nameof(dataFlowName));
}
if (dataFlow == null)
{
throw new ArgumentNullException(nameof(dataFlow));
}
using var message = CreateCreateOrUpdateDataFlowRequest(dataFlowName, dataFlow, ifMatch);
await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false);
switch (message.Response.Status)
{
case 200:
case 202:
return(message.Response);
default:
throw await ClientDiagnostics.CreateRequestFailedExceptionAsync(message.Response).ConfigureAwait(false);
}
}
private static void ValidateDataFlow(DataFactoryManagementClient client, string resourceGroupName, string dataFactoryName, DataFlowResource expected, DataFlowResource actual, string expectedName)
{
ValidateSubResource(client, resourceGroupName, actual, dataFactoryName, expectedName, "dataflows");
if (string.Equals(expectedName, "TestPowerQuery", StringComparison.InvariantCultureIgnoreCase))
{
Assert.IsType <WranglingDataFlow>(actual.Properties);
}
else
{
Assert.IsType <MappingDataFlow>(actual.Properties);
}
}
internal HttpMessage CreateCreateOrUpdateDataFlowRequest(string dataFlowName, DataFlowResource dataFlow, string ifMatch)
{
var message = _pipeline.CreateMessage();
var request = message.Request;
request.Method = RequestMethod.Put;
var uri = new RawRequestUriBuilder();
uri.AppendRaw(endpoint, false);
uri.AppendPath("/dataflows/", false);
uri.AppendPath(dataFlowName, true);
uri.AppendQuery("api-version", apiVersion, true);
request.Uri = uri;
if (ifMatch != null)
{
request.Headers.Add("If-Match", ifMatch);
}
request.Headers.Add("Accept", "application/json");
request.Headers.Add("Content-Type", "application/json");
var content = new Utf8JsonRequestContent();
content.JsonWriter.WriteObjectValue(dataFlow);
request.Content = content;
return(message);
}
private DisposableDataFlow(DataFlowClient client, DataFlowResource resource)
{
_client = client;
Resource = resource;
}
