public PSDataFactory(DataFactory dataFactory)
{
if (dataFactory == null)
{
throw new ArgumentNullException("dataFactory");
}
this.dataFactory = dataFactory;
}
public void CanCreateDataFactory()
{
// Arrange
DataFactory expected = new DataFactory()
{
Name = DataFactoryName,
Location = Location,
Properties = new DataFactoryProperties() { ProvisioningState = "Succeeded" }
};
dataFactoriesClientMock.Setup(
f =>
f.CreatePSDataFactory(
It.Is<CreatePSDataFactoryParameters>(
parameters =>
parameters.ResourceGroupName == ResourceGroupName &&
parameters.DataFactoryName == DataFactoryName &&
parameters.Location == Location)))
.CallBase()
.Verifiable();
dataFactoriesClientMock.Setup(
f => f.CreateOrUpdateDataFactory(ResourceGroupName, DataFactoryName, Location, tags))
.Returns(expected)
.Verifiable();
// Action
cmdlet.Tags = tags.ToHashtable();
cmdlet.Force = true;
cmdlet.ExecuteCmdlet();
// Assert
dataFactoriesClientMock.VerifyAll();
commandRuntimeMock.Verify(
f =>
f.WriteObject(
It.Is<PSDataFactory>(
df =>
df.DataFactoryName == expected.Name &&
df.Location == expected.Location)),
Times.Once());
}
public void OAuthAuthorizationSessionTest()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
const string oAuthType = "AzureDataLake";
string resourceGroupName = TestUtilities.GenerateName("resourcegroup");
string factoryName = TestUtilities.GenerateName("DataFactory");
string serverLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetDataFactoryManagementClient(handler);
ResourceGroup resourceGroup = new ResourceGroup() { Location = serverLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// create a data factory
var df = new Microsoft.Azure.Management.DataFactories.Models.DataFactory()
{
Name = factoryName,
Location = serverLocation
};
client.DataFactories.CreateOrUpdate(resourceGroupName, new DataFactoryCreateOrUpdateParameters()
{
DataFactory = df,
});
AuthorizationSessionGetResponse response = client.OAuth.Get(resourceGroupName, factoryName, oAuthType);
Assert.True(response.StatusCode == HttpStatusCode.OK);
Assert.NotNull(response.AuthorizationSession.Endpoint);
Assert.NotNull(response.AuthorizationSession.SessionId);
}
}
public void CanThrowIfDataFactoryProvisioningFailed()
{
// Arrange
DataFactory expected = new DataFactory()
{
Name = DataFactoryName,
Location = Location,
Properties = new DataFactoryProperties() { ProvisioningState = "Failed" }
};
dataFactoriesClientMock.Setup(
f =>
f.CreatePSDataFactory(
It.Is<CreatePSDataFactoryParameters>(
parameters =>
parameters.ResourceGroupName == ResourceGroupName &&
parameters.DataFactoryName == DataFactoryName &&
parameters.Location == Location)))
.CallBase()
.Verifiable();
dataFactoriesClientMock.Setup(
f => f.CreateOrUpdateDataFactory(ResourceGroupName, DataFactoryName, Location, tags))
.Returns(expected)
.Verifiable();
// Action
cmdlet.Tags = tags.ToHashtable();
cmdlet.Force = true;
// Assert
Assert.Throws<ProvisioningFailedException>(() => cmdlet.ExecuteCmdlet());
}
/// <summary>
/// Serializes the given <see cref="DataFactory" /> into JSON, by mocking a create request to
/// exercise the client's serialization logic.
/// </summary>
/// <param name="item">The object to serialize.</param>
/// <returns></returns>
internal static string SerializeDataFactoryToJson(DataFactory item)
{
var createParams = new DataFactoryCreateOrUpdateParameters() { DataFactory = item };
var handler = new MockResourceProviderDelegatingHandler();
var client = GetFakeClient(handler);
string resourceGroupName = Guid.NewGuid().ToString("D");
client.DataFactories.BeginCreateOrUpdate(resourceGroupName, createParams);
return handler.Json;
}
public void TestCreateTable()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
string resourceGroupName = TestUtilities.GenerateName("resourcegroup");
string factoryName = TestUtilities.GenerateName("DataFactory");
string linkedServiceName = "foo2"; // need to hard coded as it is referenced in table json
string tableName = TestUtilities.GenerateName("table");
string serverLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetDataPipelineManagementClient(handler);
try
{
ResourceGroup resourceGroup = new ResourceGroup() { Location = serverLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// create a data factory
DataFactory df = new DataFactory() { Name = factoryName, Location = serverLocation };
client.DataFactories.CreateOrUpdate(resourceGroupName, new DataFactoryCreateOrUpdateParameters()
{
DataFactory = df,
});
// verify data factory
var dfResponse = client.DataFactories.Get(resourceGroupName, factoryName);
Assert.True(dfResponse.StatusCode == HttpStatusCode.OK);
Assert.True(dfResponse.DataFactory.Name == df.Name);
Assert.True(dfResponse.DataFactory.Location == df.Location);
// create a linked service
string content = File.ReadAllText(@"Resources\linkedService.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, linkedServiceName, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
// verify linked service
var lsResponse = client.LinkedServices.Get(resourceGroupName, factoryName, linkedServiceName);
Assert.True(lsResponse.StatusCode == HttpStatusCode.OK);
Assert.True(lsResponse.LinkedService.Name == linkedServiceName);
// create a table
content = File.ReadAllText(@"Resources\table.json");
client.Tables.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, tableName, new TableCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
// verify table
var tResponse = client.Tables.Get(resourceGroupName, factoryName, tableName);
Assert.True(tResponse.StatusCode == HttpStatusCode.OK);
Assert.True(tResponse.Table.Name == tableName);
}
finally
{
client.DataFactories.Delete(resourceGroupName, factoryName);
resourceClient.ResourceGroups.Delete(resourceGroupName);
}
}
}
public PSDataFactory()
{
dataFactory = new DataFactory();
}
public void WikipediaPipelineE2ETest()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
// Linked Service names are hard-coded as they are referenced in Tables.
// Table names are hard-coded as they are referenced in the Pipeline.
const string LinkedServiceNameClickEvents = "LinkedService-WikipediaClickEvents";
const string LinkedServiceNameCuratedWikiData = "LinkedService-CuratedWikiData";
const string LinkedServiceNameAggregatedData = "LinkedService-WikiAggregatedData";
const string LinkedServiceNameHDInsightByoc = "HDILinkedService";
const string TableNameClickEvents = "DA_WikipediaClickEvents";
const string TableNameCuratedWikiData = "DA_CuratedWikiData";
const string TableNameAggregatedData = "DA_WikiAggregatedData";
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
string resourceGroupName = TestUtilities.GenerateName("resourcegroup");
string factoryName = TestUtilities.GenerateName("DataFactory");
string serverLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetDataFactoryManagementClient(handler);
string pipelineName = TestUtilities.GenerateName("DP_Wikipedia");
ResourceGroup resourceGroup = new ResourceGroup() { Location = serverLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// create a data factory
var df = new Microsoft.Azure.Management.DataFactories.Models.DataFactory()
{
Name = factoryName,
Location = serverLocation
};
client.DataFactories.CreateOrUpdate(resourceGroupName, new DataFactoryCreateOrUpdateParameters()
{
DataFactory = df,
});
// verify data factory
var dfResponse = client.DataFactories.Get(resourceGroupName, factoryName);
Assert.True(dfResponse.StatusCode == HttpStatusCode.OK);
Assert.True(dfResponse.DataFactory.Name == df.Name);
Assert.True(dfResponse.DataFactory.Location == df.Location);
// create linked services
string content = File.ReadAllText(@"Resources\LinkedService_WikipediaClickEvents.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
LinkedServiceNameClickEvents,
new Microsoft.Azure.Management.DataFactories.Core.Models.
LinkedServiceCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
content = File.ReadAllText(@"Resources\LinkedService_CuratedWikiData.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
LinkedServiceNameCuratedWikiData,
new Microsoft.Azure.Management.DataFactories.Core.Models.
LinkedServiceCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
content = File.ReadAllText(@"Resources\LinkedService_WikiAggregatedData.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
LinkedServiceNameAggregatedData,
new Microsoft.Azure.Management.DataFactories.Core.Models.
LinkedServiceCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
content = File.ReadAllText(@"Resources\LinkedService_HDIBYOC.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
LinkedServiceNameHDInsightByoc,
new Microsoft.Azure.Management.DataFactories.Core.Models.
LinkedServiceCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
// Create Datasets
content = File.ReadAllText(@"Resources\DA_WikiAggregatedData.json");
client.Datasets.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
TableNameAggregatedData,
new Microsoft.Azure.Management.DataFactories.Core.Models.
DatasetCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
content = File.ReadAllText(@"Resources\DA_CuratedWikiData.json");
client.Datasets.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
TableNameCuratedWikiData,
new Microsoft.Azure.Management.DataFactories.Core.Models.
DatasetCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
content = File.ReadAllText(@"Resources\DA_WikipediaClickEvents.json");
client.Datasets.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
TableNameClickEvents,
new Microsoft.Azure.Management.DataFactories.Core.Models.
DatasetCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
// create pipeline
content = File.ReadAllText(@"Resources\DP_Wikisamplev2json.json");
client.Pipelines.CreateOrUpdateWithRawJsonContent(
resourceGroupName,
factoryName,
pipelineName,
new Microsoft.Azure.Management.DataFactories.Core.Models.
PipelineCreateOrUpdateWithRawJsonContentParameters() { Content = content, });
DateTime now = DateTime.Parse("2014-10-08T12:00:00");
DateTime start = new DateTime(now.Year, now.Month, now.Day, now.Hour - 4, 0, 0);
string startTime = start.ToString("yyyy-MM-ddTHH:mm:ss");
string endTime = start.AddHours(2).ToString("yyyy-MM-ddTHH:mm:ss");
client.Pipelines.SetActivePeriod(resourceGroupName, factoryName, pipelineName,
new PipelineSetActivePeriodParameters()
{
ActivePeriodStartTime = startTime,
ActivePeriodEndTime = endTime,
});
// verify linked services
string[] linkedServices = new string[]
{
LinkedServiceNameClickEvents, LinkedServiceNameCuratedWikiData, LinkedServiceNameAggregatedData,
LinkedServiceNameHDInsightByoc
};
foreach (string s in linkedServices)
{
var lsResponse = client.LinkedServices.Get(resourceGroupName, factoryName, s);
Assert.True(lsResponse.StatusCode == HttpStatusCode.OK);
Assert.True(lsResponse.LinkedService.Name == s);
}
// verify tables
string[] tableNames = new string[] { TableNameClickEvents, TableNameCuratedWikiData, TableNameAggregatedData };
foreach (string tableName in tableNames)
{
var tResponse = client.Datasets.Get(resourceGroupName, factoryName, tableName);
Assert.True(tResponse.StatusCode == HttpStatusCode.OK);
Assert.True(tResponse.Dataset.Name == tableName);
}
// verify slice
var sliceResponse = client.DataSlices.List(resourceGroupName, factoryName, TableNameAggregatedData,
new DataSliceListParameters(startTime, endTime));
Assert.True(sliceResponse.StatusCode == HttpStatusCode.OK);
Assert.True(sliceResponse.DataSlices.Count == 2);
// verify list and get slice run
foreach (var slice in sliceResponse.DataSlices)
{
var listSliceRunResponse = client.DataSliceRuns.List(resourceGroupName, factoryName,
TableNameAggregatedData, new DataSliceRunListParameters(slice.Start.ConvertToISO8601DateTimeString()));
Assert.True(listSliceRunResponse.StatusCode == HttpStatusCode.OK);
foreach (var dataSliceRun in listSliceRunResponse.DataSliceRuns)
{
var getSliceRunResponse = client.DataSliceRuns.Get(resourceGroupName, factoryName,
dataSliceRun.Id);
Assert.True(getSliceRunResponse.StatusCode == HttpStatusCode.OK);
}
}
}
}
public void SqlCopyPipelineTest()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
string resourceGroupName = TestUtilities.GenerateName("resourcegroup");
string factoryName = TestUtilities.GenerateName("DataFactory");
string serverLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetDataPipelineManagementClient(handler);
string pipelineName = "DataPipeline-Sql2SqlTest";
string linkedServiceName = "LinkedService-AzureSql";
string sourceTableName = "Table-Sql2Sql-Source";
string sinkTableName = "Table-Sql2Sql-Sink";
ResourceGroup resourceGroup = new ResourceGroup() { Location = serverLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// create a data factory
DataFactory df = new DataFactory() { Name = factoryName, Location = serverLocation };
client.DataFactories.CreateOrUpdate(resourceGroupName, new DataFactoryCreateOrUpdateParameters()
{
DataFactory = df,
});
// verify data factory
var dfResponse = client.DataFactories.Get(resourceGroupName, factoryName);
Assert.True(dfResponse.StatusCode == HttpStatusCode.OK);
Assert.True(dfResponse.DataFactory.Name == df.Name);
Assert.True(dfResponse.DataFactory.Location == df.Location);
// create linked service
string content = File.ReadAllText(@"Resources\SqlStoredProcedureParameters\LinkedService_SqlAzure.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, linkedServiceName, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
// Create Tables
content = File.ReadAllText(@"Resources\SqlStoredProcedureParameters\Table_SourceTable.json");
client.Tables.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, sourceTableName, new TableCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
content = File.ReadAllText(@"Resources\SqlStoredProcedureParameters\Table_SinkTable.json");
client.Tables.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, sinkTableName, new TableCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
// create pipeline
content = File.ReadAllText(@"Resources\SqlStoredProcedureParameters\Pipeline_Copy.json");
client.Pipelines.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, pipelineName, new PipelineCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
string startTime = "2015-02-18T12:00:00Z";
string endTime = "2015-02-18T14:00:00Z";
client.Pipelines.SetActivePeriod(resourceGroupName, factoryName, pipelineName, new PipelineSetActivePeriodParameters()
{
ActivePeriodStartTime = startTime,
ActivePeriodEndTime = endTime,
});
// verify linked services
string[] linkedServices = new string[] { linkedServiceName };
foreach (string s in linkedServices)
{
var lsResponse = client.LinkedServices.Get(resourceGroupName, factoryName, s);
Assert.True(lsResponse.StatusCode == HttpStatusCode.OK);
Assert.True(lsResponse.LinkedService.Name == s);
}
// verify tables
string[] tableNames = new string[] { sourceTableName, sinkTableName };
foreach (string tableName in tableNames)
{
var tResponse = client.Tables.Get(resourceGroupName, factoryName, tableName);
Assert.True(tResponse.StatusCode == HttpStatusCode.OK);
Assert.True(tResponse.Table.Name == tableName);
}
// verify slice
var sliceResponse = client.DataSlices.List(resourceGroupName, factoryName, sinkTableName, startTime, endTime);
Assert.True(sliceResponse.StatusCode == HttpStatusCode.OK);
Assert.True(sliceResponse.DataSlices.Count == 2);
}
}
public void WikipediaPipelineTest()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
string resourceGroupName = TestUtilities.GenerateName("resourcegroup");
string factoryName = TestUtilities.GenerateName("DataFactory");
string serverLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetDataPipelineManagementClient(handler);
// Linked Service names are hard-coded as they are referenced in Data Artifacts.
// Data Artifact names are hard-coded as they are referenced in Data Pipeline.
string linkedService_ClickEvent = "LinkedService-WikipediaClickEvents";
string linkedService_CuratedData = "LinkedService-CuratedWikiData";
string linkedService_AggregatedData = "LinkedService-WikiAggregatedData";
string linkedService_Hdibyoc = "HDILinkedService";
string dataArtifact_ClickEvent = "DA_WikipediaClickEvents";
string dataArtifact_CuratedData = "DA_CuratedWikiData";
string dataArtifact_AggregatedData = "DA_WikiAggregatedData";
string pipelineName = TestUtilities.GenerateName("DP_Wikipedia");
ResourceGroup resourceGroup = new ResourceGroup() { Location = serverLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// create a data factory
DataFactory df = new DataFactory() { Name = factoryName, Location = serverLocation };
client.DataFactories.CreateOrUpdate(resourceGroupName, new DataFactoryCreateOrUpdateParameters()
{
DataFactory = df,
});
// verify data factory
var dfResponse = client.DataFactories.Get(resourceGroupName, factoryName);
Assert.True(dfResponse.StatusCode == HttpStatusCode.OK);
Assert.True(dfResponse.DataFactory.Name == df.Name);
Assert.True(dfResponse.DataFactory.Location == df.Location);
// create linked services
string content = File.ReadAllText(@"Resources\LinkedService_WikipediaClickEvents.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, linkedService_ClickEvent, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
content = File.ReadAllText(@"Resources\LinkedService_CuratedWikiData.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, linkedService_CuratedData, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
content = File.ReadAllText(@"Resources\LinkedService_WikiAggregatedData.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, linkedService_AggregatedData, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
content = File.ReadAllText(@"Resources\LinkedService_HDIBYOC.json");
client.LinkedServices.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, linkedService_Hdibyoc, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
// Create Tables
content = File.ReadAllText(@"Resources\DA_WikiAggregatedData.json");
client.Tables.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, dataArtifact_AggregatedData, new TableCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
content = File.ReadAllText(@"Resources\DA_CuratedWikiData.json");
client.Tables.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, dataArtifact_CuratedData, new TableCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
content = File.ReadAllText(@"Resources\DA_WikipediaClickEvents.json");
client.Tables.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, dataArtifact_ClickEvent, new TableCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
// create pipeline
content = File.ReadAllText(@"Resources\DP_Wikisamplev2json.json");
client.Pipelines.CreateOrUpdateWithRawJsonContent(resourceGroupName, factoryName, pipelineName, new PipelineCreateOrUpdateWithRawJsonContentParameters()
{
Content = content,
});
DateTime now = DateTime.Parse("2014-10-08T12:00:00");
DateTime start = new DateTime(now.Year, now.Month, now.Day, now.Hour - 4, 0, 0);
string startTime = start.ToString("yyyy-MM-ddTHH:mm:ss");
string endTime = start.AddHours(2).ToString("yyyy-MM-ddTHH:mm:ss");
client.Pipelines.SetActivePeriod(resourceGroupName, factoryName, pipelineName, new PipelineSetActivePeriodParameters()
{
ActivePeriodStartTime = startTime,
ActivePeriodEndTime = endTime,
});
// verify linked services
string[] linkedServices = new string[] { linkedService_ClickEvent, linkedService_CuratedData, linkedService_AggregatedData, linkedService_Hdibyoc };
foreach (string s in linkedServices)
{
var lsResponse = client.LinkedServices.Get(resourceGroupName, factoryName, s);
Assert.True(lsResponse.StatusCode == HttpStatusCode.OK);
Assert.True(lsResponse.LinkedService.Name == s);
}
// verify tables
string[] tableNames = new string[] { dataArtifact_ClickEvent, dataArtifact_CuratedData, dataArtifact_AggregatedData };
foreach (string tableName in tableNames)
{
var tResponse = client.Tables.Get(resourceGroupName, factoryName, tableName);
Assert.True(tResponse.StatusCode == HttpStatusCode.OK);
Assert.True(tResponse.Table.Name == tableName);
}
// verify slice
var sliceResponse = client.DataSlices.List(resourceGroupName, factoryName, dataArtifact_AggregatedData, startTime, endTime);
Assert.True(sliceResponse.StatusCode == HttpStatusCode.OK);
Assert.True(sliceResponse.DataSlices.Count == 2);
}
}