public void TestInitParameters() { // uninitialized Params List <DfModels.LinkedService> linkedServices = null; List <DfModels.Dataset> datasets = null; DfModels.Activity activity = null; DfRuntime.IActivityLogger logger = new ActivityLogger(); // testdata var pipelinePath = Path.GetFullPath(@"..\..\Data\logs-etl-pipeline.json"); var activityName = "stage"; Assert.IsTrue(File.Exists(pipelinePath)); // do run the wizard Wizard.InitParameters(pipelinePath, activityName, out linkedServices, out datasets, out activity); Assert.AreEqual(2, linkedServices.Count); Assert.AreEqual(3, datasets.Count); DfModels.Dataset importDataset = datasets.Where(dataset => dataset.Name == "import-month-dataset").Single(); Assert.IsInstanceOfType(importDataset.Properties.TypeProperties, typeof(DfModels.AzureBlobDataset)); DfModels.Dataset factDataset = datasets.Where(dataset => dataset.Name == "fact-month-dataset").Single(); Assert.IsInstanceOfType(factDataset.Properties.TypeProperties, typeof(DfModels.AzureBlobDataset)); DfModels.Dataset factTable = datasets.Where(dataset => dataset.Name == "fact-month-table").Single(); Assert.IsInstanceOfType(factTable.Properties.TypeProperties, typeof(DfModels.AzureTableDataset)); // run the activity DfRuntime.IDotNetActivity testedActivity = new TestActivity(); testedActivity.Execute(linkedServices, datasets, activity, logger); }
public BlobLocation(LinkedService linkedService, Dataset dataset, string sliceYear, string sliceMonth, string sliceDay) { _linkedService = linkedService; _dataset = dataset; _sliceYear = sliceYear; _sliceMonth = sliceMonth; _sliceDay = sliceDay; }
public PSDataset(Dataset dataset) { if (dataset == null) { throw new ArgumentNullException("dataset"); } if (dataset.Properties == null) { dataset.Properties = new DatasetProperties(); } this.dataset = dataset; }
// ToDo: enable the tests when we can set readonly provisioning state in test //[Fact] //[Trait(Category.AcceptanceType, Category.CheckIn)] public void CanCreateDataset() { // Arrange Dataset expected = new Dataset() { Name = datasetName, Properties = new DatasetProperties() }; dataFactoriesClientMock.Setup(c => c.ReadJsonFileContent(It.IsAny<string>())) .Returns(rawJsonContent) .Verifiable(); dataFactoriesClientMock.Setup( c => c.CreatePSDataset( It.Is<CreatePSDatasetParameters>( parameters => parameters.Name == datasetName && parameters.ResourceGroupName == ResourceGroupName && parameters.DataFactoryName == DataFactoryName))) .CallBase() .Verifiable(); dataFactoriesClientMock.Setup( c => c.CreateOrUpdateDataset(ResourceGroupName, DataFactoryName, datasetName, rawJsonContent)) .Returns(expected) .Verifiable(); // Action cmdlet.File = filePath; cmdlet.Force = true; cmdlet.ExecuteCmdlet(); // Assert dataFactoriesClientMock.VerifyAll(); commandRuntimeMock.Verify( f => f.WriteObject( It.Is<PSDataset>( tbl => ResourceGroupName == tbl.ResourceGroupName && DataFactoryName == tbl.DataFactoryName && expected.Name == tbl.DatasetName)), Times.Once()); }
public CustomDbDatasetProvider(Dataset dataset, LinkedService linkedService, Func<string, LinkedService> linkedServiceResolver) { var props = dataset.Properties.TypeProperties as CustomDataset; var packageLnkedServiceName = props.ServiceExtraProperties["packageLinkedService"].ToString(); Dataset = new CustomDbDataset { InstanceName = props.ServiceExtraProperties["instanceName"].ToString(), DbContextName = props.ServiceExtraProperties["dbContextName"].ToString(), AssemblyFile = props.ServiceExtraProperties["assemblyFile"].ToString() }; InstanceName = Dataset.InstanceName; var blob = Helpers.GetBlob(linkedServiceResolver(packageLnkedServiceName), Dataset.AssemblyFile); var path = Path.Combine(Environment.CurrentDirectory, Path.GetFileName(Dataset.AssemblyFile)); blob.DownloadToFile(path, FileMode.Create); var m = Assembly.LoadFrom(path); Context = (DbContext)Activator.CreateInstance(m.GetType(Dataset.DbContextName), (linkedService.Properties.TypeProperties as AzureSqlDatabaseLinkedService).ConnectionString); }
public AzureBlobProvider(Dataset dataset, LinkedService linkedService, Slice slice) { InstanceName = dataset.Name; Structure = (dataset.Properties as DatasetProperties).Structure; var azblobDataset = dataset.Properties.TypeProperties as AzureBlobDataset; var filepath = Path.Combine(azblobDataset.FolderPath, azblobDataset.FileName); filepath = Helpers.ReplaceByPatition(filepath, azblobDataset.PartitionedBy, slice); Blob = Helpers.GetBlob(linkedService, filepath); var format = azblobDataset.Format as TextFormat; if (format != null) Configuration = new CsvConfiguration { Delimiter = format.ColumnDelimiter, Encoding = Encoding.GetEncoding(format.EncodingName) }; else Configuration = new CsvConfiguration { }; }
public CustomAzureBlobProvider(Dataset dataset, LinkedService linkedService, Slice slice) { var props = dataset.Properties.TypeProperties as CustomDataset; if (props.ServiceExtraProperties.ContainsKey("instanceName")) InstanceName = props.ServiceExtraProperties["instanceName"].ToString(); var enc = props.ServiceExtraProperties.ContainsKey("encoding") ? props.ServiceExtraProperties["encoding"].ToString() : null; if (!string.IsNullOrEmpty(enc)) { try { Encoding = Encoding.GetEncoding(enc); } catch (ArgumentException){} } var filepath = props.ServiceExtraProperties["filePath"].ToString(); var regex = new Regex(@"\{(.+?)\}"); var matches = regex.Matches(filepath); foreach(Match m in matches) { var key = m.Groups[1].Value; if (props.ServiceExtraProperties.ContainsKey(key)) { var formatAndName = props.ServiceExtraProperties[key].ToString().Split(',').Select(_ => _.Trim()).ToArray(); var value = formatAndName[1].ToLower() == "slicestart" ? slice.Start.ToString(formatAndName[0]) : formatAndName[1].ToLower() == "sliceend" ? slice.End.ToString(formatAndName[0]) : ""; filepath = filepath.Replace($"{{{formatAndName[1]}}}", value); } } Blob = Helpers.GetBlob(linkedService, filepath); var format = props.ServiceExtraProperties["format"]; Format = format == null ? CustomAzureBlobFormat.Unknown : format.ToString().ToLower() == "json" ? CustomAzureBlobFormat.Json : CustomAzureBlobFormat.Unknown; if (props.ServiceExtraProperties.ContainsKey("test")) Console.Write(props.ServiceExtraProperties["test"].ToString()); }
public static bool IsMatch(Dataset dataset, LinkedService linkedService) { return dataset.Properties.Type == "CustomDataset" && linkedService.Properties.Type == "AzureSqlDatabase"; }
public void CanThrowIfDatasetProvisioningFailed() { // Arrange Dataset expected = new Dataset() { Name = datasetName, Properties = new DatasetProperties() }; dataFactoriesClientMock.Setup(c => c.ReadJsonFileContent(It.IsAny<string>())) .Returns(rawJsonContent) .Verifiable(); dataFactoriesClientMock.Setup( c => c.CreatePSDataset( It.Is<CreatePSDatasetParameters>( parameters => parameters.Name == datasetName && parameters.ResourceGroupName == ResourceGroupName && parameters.DataFactoryName == DataFactoryName))) .CallBase() .Verifiable(); dataFactoriesClientMock.Setup( c => c.CreateOrUpdateDataset(ResourceGroupName, DataFactoryName, datasetName, rawJsonContent)) .Returns(expected) .Verifiable(); // Action cmdlet.File = filePath; cmdlet.Force = true; // Assert Assert.Throws<ProvisioningFailedException>(() => cmdlet.ExecuteCmdlet()); }
static void Main(string[] args) { var customActivity = new MongoDbDumpTransformActivity(); var config = JObject.Parse( File.ReadAllText(@"..\..\..\..\..\LocalSettings-DataFactory.json")); var linkedServices = new List<LinkedService>() { new LinkedService("GHTorrentAzureStorage", new LinkedServiceProperties(new CustomDataSourceLinkedService(JObject.Parse(String.Format("{{\"sasUri\": \"{0}\"}}", config["GHTorrentAzureStorage"][0]["value"].Value<string>()))))) ,new LinkedService("GHInsightsAzureStorage", new LinkedServiceProperties(new AzureStorageLinkedService(config["GHInsightsAzureStorage"][0]["value"].Value<string>()))) }; var mongoDbDump = new Dataset("MongoDbDump", new DatasetProperties(new AzureBlobDataset() { FolderPath = "downloads/", FileName = "mongo-dump-{Year}-{Month}-{Day}.tar.gz", PartitionedBy = new List<Partition>() { { new Partition() { Name = "Year", Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "yyyy"} } }, { new Partition() { Name = "Month", Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "MM"} } }, { new Partition() { Name = "Day", Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "dd"} } } } }, new Availability("Daily", 1), "GHTorrentAzureStorage")); var eventDetailRawFilesBlob = new Dataset("EventDetail", new DatasetProperties(new AzureBlobDataset() { FolderPath = @"test/{EventName}/v1/{Year}/{Month}", FileName = "{EventName}_{Year}_{Month}_{Day}.json.gz", PartitionedBy = new List<Partition>() { { new Partition() { Name = "Year", Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "yyyy"} } }, { new Partition() { Name = "Month", Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "MM"} } }, { new Partition() { Name = "Day", Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "dd"} } } } }, new Availability("Daily", 1), "GHInsightsAzureStorage")); var datasets = new List<Dataset>() { mongoDbDump, eventDetailRawFilesBlob }; var activity = new Activity() { Description = "Fake Activity", Inputs = new ActivityInput[] {new ActivityInput("MongoDbDump")}, LinkedServiceName = "BatchProcessor", Name = "GHTorrentEventDetailPipeline", Outputs = new ActivityOutput[] {new ActivityOutput("EventDetail") }, Policy = new ActivityPolicy() { Concurrency = 4, ExecutionPriorityOrder = "NewestFirst", Retry = 3, Timeout = TimeSpan.Parse("04:00:00") }, Scheduler = new Scheduler("Day", 1), TypeProperties = new DotNetActivity("DataFactoryLib.dll" , "DataFactoryLib.MongoDbDumpTransformActivity" , "datafactory/DataFactoryLib.zip" , "GHInsightsAzureStorage") { ExtendedProperties = new Dictionary<string, string>() { {"Year", "2015"}, {"Month", "12"}, {"Day", "02"} } } }; IActivityLogger logger = new DebugLogger(); customActivity.Execute(linkedServices, datasets, activity, logger); }
/// <summary> /// Gets the folderPath value from the input/output dataset. /// </summary> private static string GetFolderPath(Dataset dataArtifact) { if (dataArtifact == null || dataArtifact.Properties == null) { return null; } AzureBlobDataset blobDataset = dataArtifact.Properties.TypeProperties as AzureBlobDataset; if (blobDataset == null) { return null; } return blobDataset.FolderPath; }
public static bool IsMatch(Dataset dataset, LinkedService linkedService) { return dataset.Properties.Type == "CustomDataset" && linkedService.Properties.Type == "AzureStorage" && ((CustomDataset)dataset.Properties.TypeProperties).ServiceExtraProperties["type"].ToString() == "AzureBlob"; }
public PSDataset() { this.dataset = new Dataset() { Properties = new DatasetProperties() }; }
public static void InitParameters( string pipelinePath, string activityName, out List <Models.LinkedService> linkedServices, out List <Models.Dataset> datasets, out Models.Activity activity) { // init the parameters linkedServices = new List <Models.LinkedService>(); datasets = new List <Models.Dataset>(); activity = new Models.Activity(); // parse the pipeline json source var pipelineJson = File.ReadAllText(pipelinePath); var dummyPipeline = JsonConvert.DeserializeObject <Dummy.Pipeline>(pipelineJson); foreach (var dummyActivity in dummyPipeline.Properties.Activities) { // find the relevant activity in the pipeline if (dummyActivity.Name != activityName) { continue; } activity.Name = dummyActivity.Name; // get the input and output tables var dummyDatasets = new HashSet <Dummy.ActivityData>(); dummyDatasets.UnionWith(dummyActivity.Inputs); dummyDatasets.UnionWith(dummyActivity.Outputs); var dummyServices = new HashSet <Dummy.LinkedService>(); // init the data tables foreach (var dummyDataset in dummyDatasets) { // parse the table json source var dataPath = Path.Combine(Path.GetDirectoryName(pipelinePath), dummyDataset.Name + ".json"); var dataJson = File.ReadAllText(dataPath); var dummyTable = JsonConvert.DeserializeObject <Dummy.Table>(dataJson); { // initialize dataset properties Models.DatasetTypeProperties datasetProperties; switch (dummyTable.Properties.Type) { case "AzureBlob": // init the azure model var blobDataset = new Models.AzureBlobDataset(); blobDataset.FolderPath = dummyTable.Properties.TypeProperties.FolderPath; blobDataset.FileName = dummyTable.Properties.TypeProperties.FileName; datasetProperties = blobDataset; break; case "AzureTable": var tableDataset = new Models.AzureTableDataset(); tableDataset.TableName = dummyTable.Properties.TypeProperties.TableName; datasetProperties = tableDataset; break; default: throw new Exception(string.Format("Unexpected Dataset.Type {0}", dummyTable.Properties.Type)); } // initialize dataset { var dataDataset = new Models.Dataset( dummyDataset.Name, new Models.DatasetProperties( datasetProperties, new CommonModels.Availability(), "" ) ); dataDataset.Properties.LinkedServiceName = dummyTable.Properties.LinkedServiceName; datasets.Add(dataDataset); } } // register the input or output in the activity if (dummyDataset is Dummy.ActivityInput) { activity.Inputs.Add(new CommonModels.ActivityInput(dummyDataset.Name)); } if (dummyDataset is Dummy.ActivityOutput) { activity.Outputs.Add(new CommonModels.ActivityOutput(dummyDataset.Name)); } // parse the linked service json source for later use var servicePath = Path.Combine(Path.GetDirectoryName(pipelinePath), dummyTable.Properties.LinkedServiceName + ".json"); var serviceJson = File.ReadAllText(servicePath); var storageService = JsonConvert.DeserializeObject <Dummy.StorageService>(serviceJson); dummyServices.Add(storageService); } // parse the hd insight service json source { var servicePath = Path.Combine(Path.GetDirectoryName(pipelinePath), dummyActivity.LinkedServiceName + ".json"); var serviceJson = File.ReadAllText(servicePath); var computeService = JsonConvert.DeserializeObject <Dummy.ComputeService>(serviceJson); dummyServices.Add(computeService); } // init the services foreach (var dummyService in dummyServices) { Models.LinkedService linkedService = null; // init if it is a storage service if (dummyService is Dummy.StorageService) { var dummyStorageService = dummyService as Dummy.StorageService; var service = new Models.AzureStorageLinkedService(); service.ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString; linkedService = new Models.LinkedService( dummyService.Name, new Models.LinkedServiceProperties(service) ); } // init if it is a hd insight service if (dummyService is Dummy.ComputeService) { var service = new Models.HDInsightLinkedService(); linkedService = new Models.LinkedService( dummyService.Name, new Models.LinkedServiceProperties(service) ); } linkedServices.Add(linkedService); } } if (activity.Name == null) { throw new Exception(string.Format("Activity {0} not found.", activityName)); } }