public void CreateParamsAndSaveToFiles()
{
var armObject = JObject.FromObject(_arm);
GenerateAndReplaceParameters(armObject);
var parametersObject = (JObject)armObject["parameters"];
foreach (var parameter in _parameters)
{
var param = parameter;
if (param.Name.Equals(Constants.LocationParameterName, StringComparison.InvariantCultureIgnoreCase))
{
param.Properties.DefaultValue = Constants.LocationParameterDefaultValue;
}
var content = JObject.FromObject(param.Properties);
parametersObject.Add(param.Name, content);
}
File.WriteAllText(_path, armObject.ToString(Formatting.Indented));
var armTemplateParameters = new ArmTemplateParameters
{
Parameters = _parameters.Select(i => new ArmTemplateParameterItem(i)).ToList()
};
var paramsFilePath = _path.Substring(0, _path.Length - 4) + "parameters.json";
var paramsFileJson = AdfSerializer.Serialize(armTemplateParameters);
File.WriteAllText(paramsFilePath, paramsFileJson);
}
public void AdfSerializer_ShouldParse_ScriptPath()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.HDInsightMapReduce);
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <HDInsightMapReduceTypeProperties>();
props.ClassName.ShouldNotBeNullOrWhiteSpace();
props.JarFilePath.ShouldNotBeNullOrWhiteSpace();
props.JarLinkedService.ShouldNotBeNullOrWhiteSpace();
props.Arguments.ShouldNotBeEmpty();
foreach (var param in props.Arguments)
{
param.ShouldNotBeNullOrWhiteSpace();
}
}
public void AdfSerializer_ShouldParse_AllProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var service = result.value as HDInsightOnDemand;
// Assert
service.Name.ShouldNotBeNullOrWhiteSpace();
service.Properties.Type.ShouldBe(LinkedServiceType.HDInsightOnDemand);
var props = service.Properties.TypeProperties.ShouldBeAssignableTo <HDInsightOnDemandTypeProperties>();
props.ClusterSize.ShouldBeGreaterThanOrEqualTo(1);
props.TimeToLive.ShouldBeGreaterThan(TimeSpan.FromSeconds(1));
props.Version.ShouldNotBeNullOrWhiteSpace();
props.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
props.AdditionalLinkedServiceNames.ShouldNotBeEmpty();
props.OsType.ShouldNotBeNullOrWhiteSpace();
props.HcatalogLinkedServiceName.ShouldNotBeNullOrWhiteSpace();
props.CoreConfiguration.ShouldNotBeNull();
props.HBaseConfiguration.ShouldNotBeNull();
props.HdfsConfiguration.ShouldNotBeNull();
props.HiveConfiguration.ShouldNotBeNull();
props.MapReduceConfiguration.ShouldNotBeNull();
props.OozieConfiguration.ShouldNotBeNull();
props.StormConfiguration.ShouldNotBeNull();
props.YarnConfiguration.ShouldNotBeNull();
props.HeadNodeSize.ShouldNotBeNullOrWhiteSpace();
props.DataNodeSize.ShouldNotBeNullOrWhiteSpace();
props.ZookeeperNodeSize.ShouldNotBeNullOrWhiteSpace();
}
public void AdfSerializer_ShouldParse_InlineScriptFullVersion()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullInlineScriptFile);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.DataLakeAnalyticsUSQL);
activity.Inputs.ShouldNotBeEmpty();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <DataLakeAnalyticsUsqlTypeProperties>();
props.Script.ShouldNotBeNullOrWhiteSpace();
props.ScriptPath.ShouldBeNullOrWhiteSpace();
props.ScriptLinkedService.ShouldBeNullOrWhiteSpace();
props.DegreeOfParallelism.ShouldNotBeNull();
props.Priority.ShouldNotBeNull();
props.RuntimeVersion.ShouldNotBeNullOrWhiteSpace();
props.CompilationMode.ShouldNotBeNullOrWhiteSpace();
foreach (var param in props.Parameters)
{
param.Key.ShouldNotBeNullOrWhiteSpace();
param.Value.ShouldNotBeNullOrWhiteSpace();
}
}
public void AdfSerializer_ShouldParse_InlineScript()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(ScriptInlineFile);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.HDInsightHive);
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <HDInsightHiveTypeProperties>();
props.Script.ShouldNotBeNullOrWhiteSpace();
props.ScriptPath.ShouldBeNullOrWhiteSpace();
props.ScriptLinkedService.ShouldBeNullOrWhiteSpace();
props.Defines.ShouldNotBeEmpty();
foreach (var param in props.Defines)
{
param.Key.ShouldNotBeNullOrWhiteSpace();
param.Value.ShouldNotBeNullOrWhiteSpace();
}
}
public void AdfSerializer_ShouldParse_ScriptPath()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.HDInsightStreaming);
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo<HDInsightStreamingTypeProperties>();
props.Mapper.ShouldNotBeNullOrWhiteSpace();
props.Reducer.ShouldNotBeNullOrWhiteSpace();
props.Input.ShouldNotBeNullOrWhiteSpace();
props.Output.ShouldNotBeNullOrWhiteSpace();
props.FileLinkedService.ShouldNotBeNullOrWhiteSpace();
props.GetDebugInfo.ShouldNotBeNull();
props.FilePaths.ShouldNotBeEmpty();
foreach (var param in props.FilePaths)
param.ShouldNotBeNullOrWhiteSpace();
props.Arguments.ShouldNotBeEmpty();
foreach (var param in props.Arguments)
param.ShouldNotBeNullOrWhiteSpace();
}
public void AdfSerializer_ShouldParse_AllProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.SqlServerStoredProcedure);
activity.Inputs.ShouldNotBeEmpty();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <SqlServerStoredProcedureTypeProperties>();
props.StoredProcedureName.ShouldNotBeNullOrWhiteSpace();
props.StoredProcedureParameters.ShouldNotBeEmpty();
foreach (var param in props.StoredProcedureParameters)
{
param.Key.ShouldNotBeNullOrWhiteSpace();
param.Value.ShouldNotBeNullOrWhiteSpace();
}
}
public void AdfSerializer_ShouldParse_MinimumSetOfProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(MinimumFilePath);
var pipeline = result.value as Pipeline;
// Assert
pipeline.Name.ShouldNotBeNullOrWhiteSpace();
pipeline.Properties.Description.ShouldBeNullOrWhiteSpace();
pipeline.Properties.Start.ShouldBeNull();
pipeline.Properties.End.ShouldBeNull();
pipeline.Properties.IsPaused.ShouldBeNull();
pipeline.Properties.PipelineMode.ShouldBeNull();
pipeline.Properties.ExpirationTime.ShouldBeNull();
var activities = pipeline.Properties.Activities.ShouldBeAssignableTo <Activity[]>();
var activity = activities[0];
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Description.ShouldBeNullOrWhiteSpace();
activity.Type.ShouldNotBe(ActivityType.Copy, "Sample should not contain a default Copy value");
activity.Inputs.ShouldBeNull();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldBeNullOrWhiteSpace();
activity.Policy.ShouldBeNull();
activity.Scheduler.ShouldBeNull();
}
public void AdfSerializer_ShouldParse_AllProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.DotNetActivity);
activity.Inputs.ShouldNotBeEmpty();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <DotNetActivityTypeProperties>();
props.AssemblyName.ShouldNotBeNullOrWhiteSpace();
props.EntryPoint.ShouldNotBeNullOrWhiteSpace();
props.PackageLinkedService.ShouldNotBeNullOrWhiteSpace();
props.PackageFile.ShouldNotBeNullOrWhiteSpace();
props.ExtendedProperties.ShouldNotBeEmpty();
foreach (var param in props.ExtendedProperties)
{
param.Key.ShouldNotBeNullOrWhiteSpace();
param.Value.ShouldNotBeNullOrWhiteSpace();
}
}
public void AdfSerializer_ShouldParse_ActivityWithoutInputs()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(NoInputFilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.AzureMLBatchExecution);
activity.Inputs.ShouldBeNull();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <MlBatchExecutorTypeProperties>();
props.WebServiceInput.ShouldBeNullOrWhiteSpace();
props.WebServiceInputs.ShouldBeNull();
props.WebServiceOutputs.Length.ShouldBe(activity.Outputs.Length);
foreach (var param in props.WebServiceOutputs)
{
param.Key.ShouldNotBeNullOrWhiteSpace();
param.Value.ShouldNotBeNullOrWhiteSpace();
}
props.GlobalParameters.ShouldBeNull();
}
public void AdfSerializer_ShouldParse_AllProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Inputs.ShouldNotBeEmpty();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <CopyTypeProperties>();
var sink = props.Sink.ShouldBeAssignableTo <CopySinkAzureSqlDwTable>();
sink.Type.ShouldBe(CopySinkType.SqlDWSink);
sink.WriteBatchSize.ShouldNotBeNull();
sink.WriteBatchTimeout.ShouldNotBeNull();
sink.AllowPolyBase.ShouldBe(true);
sink.SqlWriterCleanupScript.ShouldNotBeNullOrWhiteSpace();
sink.PolyBaseSettings.RejectValue.ShouldNotBeNull();
sink.PolyBaseSettings.RejectType.ShouldNotBeNullOrWhiteSpace();
sink.PolyBaseSettings.RejectSampleValue.ShouldNotBeNull();
sink.PolyBaseSettings.UseTypeDefault.ShouldNotBeNull();
}
public void AdfSerializer_ShouldParse_MinSetOfProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(MinFilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Inputs.ShouldNotBeEmpty();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <CopyTypeProperties>();
var sink = props.Sink.ShouldBeAssignableTo <CopySinkAzureSql>();
sink.Type.ShouldBe(CopySinkType.SqlSink);
sink.WriteBatchSize.ShouldBeNull();
sink.WriteBatchTimeout.ShouldBeNull();
sink.SqlWriterCleanupScript.ShouldBeNull();
sink.SqlWriterStoredProcedureName.ShouldBeNull();
sink.SqlWriterTableType.ShouldBeNull();
sink.StoredProcedureParameters.ShouldBeNull();
sink.SliceIdentifierColumnName.ShouldBeNull();
}
public void AdfSerializer_ShouldParse_Format()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var props = (result.value as AzureBlob).Properties.TypeProperties as AzureBlobTypeProperties;
// Assert
var format = props.Format.ShouldBeAssignableTo <FormatType>();
format.Type.ShouldNotBe(FormatTypes.TextFormat, "Samples should not contain default value");
format.ColumnDelimiter.ShouldNotBeNullOrWhiteSpace();
format.RowDelimiter.ShouldNotBeNullOrEmpty();
format.EscapeChar.ShouldNotBeNullOrEmpty();
format.QuoteChar.ShouldNotBeNullOrEmpty();
format.NullValue.ShouldNotBeNullOrWhiteSpace();
format.EncodingName.ShouldNotBeNullOrWhiteSpace();
format.FirstRowAsHeader.ShouldNotBeNull();
format.SkipLineCount.ShouldNotBeNull();
format.TreatEmptyAsNull.ShouldNotBeNull();
format.FilePattern.ShouldNotBeNullOrWhiteSpace();
format.JsonNodeReference.ShouldNotBeNullOrWhiteSpace();
format.JsonPathDefinition.ShouldNotBeNullOrWhiteSpace();
format.NestingSeparator.ShouldNotBeNull();
}
public void AdfSerializer_ShouldParse_ActivityTopLevelProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var pipeline = result.value as Pipeline;
// Assert
var activities = pipeline.Properties.Activities.ShouldBeAssignableTo <Activity[]>();
var activity = activities[0];
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Description.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldNotBe(ActivityType.Copy, "Sample should not contain a default Copy value");
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
activity.Inputs.ShouldNotBeEmpty();
foreach (var item in activity.Inputs)
{
item.Name.ShouldNotBeNullOrWhiteSpace();
}
activity.Outputs.ShouldNotBeEmpty();
foreach (var item in activity.Outputs)
{
item.Name.ShouldNotBeNullOrWhiteSpace();
}
}
public void AdfSerializer_ShouldParse_AllProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Name.ShouldNotBeNullOrWhiteSpace();
activity.Type.ShouldBe(ActivityType.HDInsightSpark);
activity.Inputs.ShouldNotBeEmpty();
activity.Outputs.ShouldNotBeEmpty();
activity.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
var props = activity.TypeProperties.ShouldBeAssignableTo <HDInsightSparkTypeProperties>();
props.RootPath.ShouldNotBeNullOrWhiteSpace();
props.EntryFilePath.ShouldNotBeNullOrWhiteSpace();
props.ClassName.ShouldNotBeNullOrWhiteSpace();
props.ProxyUser.ShouldNotBeNullOrWhiteSpace();
props.SparkJobLinkedService.ShouldNotBeNullOrWhiteSpace();
props.SparkConfig.ShouldNotBeNull();
props.Arguments.ShouldNotBeEmpty();
foreach (var param in props.Arguments)
{
param.ShouldNotBeNullOrWhiteSpace();
}
}
public void AdfSerializer_ShouldParse_AllUserAuthenticationProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullUserFilePath);
var service = result.value as AzureDataLakeAnalytics;
// Assert
service.Schema.ShouldNotBeNullOrWhiteSpace();
service.Name.ShouldNotBeNullOrWhiteSpace();
service.Properties.Type.ShouldBe(LinkedServiceType.AzureDataLakeAnalytics);
service.Properties.HubName.ShouldNotBeNullOrWhiteSpace();
var props = service.Properties.TypeProperties.ShouldBeAssignableTo <AzureDataLakeAnalyticsTypeProperties>();
props.AccountName.ShouldNotBeNullOrWhiteSpace();
props.DataLakeAnalyticsUri.ShouldNotBeNullOrWhiteSpace();
props.SubscriptionId.ShouldNotBeNullOrWhiteSpace();
props.ResourceGroupName.ShouldNotBeNullOrWhiteSpace();
props.Authorization.ShouldNotBeNullOrWhiteSpace();
props.SessionId.ShouldNotBeNullOrWhiteSpace();
props.ServicePrincipalId.ShouldBeNullOrEmpty();
props.ServicePrincipalKey.ShouldBeNullOrEmpty();
props.Tenant.ShouldBeNullOrEmpty();
}
public void AdfItemType_ShouldBe_DataSet()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
// Assert
result.type.ShouldBe(AdfItemType.DataSet);
}
public void LinkedServiceType_ShouldBe_HDInsight()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
// Assert
result.value.ShouldBeAssignableTo <HDInsight>();
}
public void LinkedServiceType_ShouldBe_AzureDataLakeAnalytics()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullServiceFilePath);
// Assert
result.value.ShouldBeAssignableTo <AzureDataLakeAnalytics>();
}
public void DataSetType_ShouldBe_AzureTable()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
// Assert
result.value.ShouldBeAssignableTo <AzureTable>();
}
public void LinkedServiceType_ShouldBe_AzureStorage()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
// Assert
result.value.ShouldBeAssignableTo <AzureStorage>();
}
public void AdfItemType_ShouldBe_Pipeline()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
// Assert
result.type.ShouldBe(AdfItemType.Pipeline);
}
public void AdfItemType_ShouldBe_LinkedService()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
// Assert
result.type.ShouldBe(AdfItemType.LinkedService);
}
public void AdfActivityType_ShouldBe_CopyTypeProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
// Assert
activity.Type.ShouldBe(ActivityType.Copy);
activity.TypeProperties.ShouldBeAssignableTo <CopyTypeProperties>();
}
public void AdfSerializer_ShouldParse_Translator()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
var props = activity.TypeProperties as CopyTypeProperties;
// Assert
props.Translator.Type.ShouldBe("TabularTranslator");
props.Translator.ColumnMappings.ShouldNotBeNullOrWhiteSpace();
}
public void AdfSerializer_ShouldParse_Compression()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var props = (result.value as AzureBlob).Properties.TypeProperties as AzureBlobTypeProperties;
// Assert
var compression = props.Compression.ShouldBeAssignableTo <Compression>();
compression.Type.ShouldNotBe(CompressionType.GZip, "Samples should not contain default GZip value");
compression.Level.ShouldNotBe(CompressionLevel.Optimal, "Samples should not contain default Optimal value");
}
public void AdfSerializer_ShouldParse_TopLevelBlobProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var dataset = result.value as AzureBlob;
// Assert
var props = dataset.Properties.TypeProperties.ShouldBeAssignableTo <AzureBlobTypeProperties>();
props.FolderPath.ShouldNotBeNullOrWhiteSpace();
props.FileName.ShouldNotBeNullOrWhiteSpace();
}
public void AdfSerializer_ShouldParse_RedirectIncompatibleRowSettings()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FilePath);
var activity = (result.value as Pipeline).Properties.Activities[0];
var props = activity.TypeProperties as CopyTypeProperties;
// Assert
props.EnableSkipIncompatibleRow.ShouldBe(true);
props.RedirectIncompatibleRowSettings.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
props.RedirectIncompatibleRowSettings.Path.ShouldNotBeNullOrWhiteSpace();
}
public void AdfSerializer_ShouldParse_AllTopLevelProperties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var dataset = result.value as Core.Models.DataSets.DataSet;
// Assert
dataset.Schema.ShouldNotBeNullOrWhiteSpace();
dataset.Name.ShouldNotBeNullOrWhiteSpace();
dataset.Properties.LinkedServiceName.ShouldNotBeNullOrWhiteSpace();
dataset.Properties.Published.ShouldNotBeNull();
dataset.Properties.External.ShouldNotBeNull();
}
public void AdfSerializer_ShouldParse_Properties()
{
// Arrange
// Act
var result = AdfSerializer.Deserialize(FullFilePath);
var dataset = result.value as AzureTable;
// Assert
dataset.Name.ShouldNotBeNullOrWhiteSpace();
dataset.Properties.Type.ShouldBe(DataSetType.AzureTable);
var props = dataset.Properties.TypeProperties.ShouldBeAssignableTo <AzureTableTypeProperties>();
props.TableName.ShouldNotBeNullOrWhiteSpace();
}
