public DataFrameFunctionsTests(SparkFixture fixture)
{
_spark = fixture.Spark;
_df = _spark
.Read()
.Json($"{TestEnvironment.ResourceDirectory}people.json");
}
public UdfComplexTypesTests(SparkFixture fixture)
{
_spark = fixture.Spark;
_df = _spark
.Read()
.Json(Path.Combine($"{TestEnvironment.ResourceDirectory}people.json"));
}
public UdfSimpleTypesTests(SparkFixture fixture)
{
_spark = fixture.Spark;
var data = new List <GenericRow>();
data.Add(new GenericRow(
new object[]
{
null,
new Date(2020, 1, 1),
new Timestamp(2020, 1, 1, 0, 0, 0, 0)
}));
data.Add(new GenericRow(
new object[]
{
30,
new Date(2020, 1, 2),
new Timestamp(2020, 1, 2, 15, 30, 30, 123456)
}));
var schema = new StructType(new List <StructField>()
{
new StructField("age", new IntegerType()),
new StructField("date", new DateType()),
new StructField("time", new TimestampType())
});
_df = _spark.CreateDataFrame(data, schema);
}
public DataFrameTests(SparkFixture fixture)
{
_spark = fixture.Spark;
_df = _spark
.Read()
.Schema("age INT, name STRING")
.Json($"{TestEnvironment.ResourceDirectory}people.json");
}
public UdfSimpleTypesTests(SparkFixture fixture)
{
_spark = fixture.Spark;
_df = _spark
.Read()
.Schema("name STRING, age INT, date DATE")
.Json(Path.Combine($"{TestEnvironment.ResourceDirectory}people.json"));
}
public HyperspaceFixture()
{
Environment.SetEnvironmentVariable(
SparkFixture.EnvironmentVariableNames.ExtraSparkSubmitArgs,
"--packages com.microsoft.hyperspace:hyperspace-core_2.11:0.1.0");
SparkFixture = new SparkFixture();
}
public DeltaFixture()
{
Environment.SetEnvironmentVariable(
SparkFixture.EnvironmentVariableNames.ExtraSparkSubmitArgs,
"--packages io.delta:delta-core_2.11:0.6.0 " +
"--conf spark.databricks.delta.snapshotPartitions=2 " +
"--conf spark.sql.sources.parallelPartitionDiscovery.parallelism=5");
SparkFixture = new SparkFixture();
}
public HyperspaceFixture()
{
Version sparkVersion = SparkSettings.Version;
string hyperspaceVersion = sparkVersion.Major switch
{
2 => "hyperspace-core_2.11:0.4.0",
3 => "hyperspace-core_2.12:0.4.0",
_ => throw new NotSupportedException($"Spark {sparkVersion} not supported.")
};
Environment.SetEnvironmentVariable(
SparkFixture.EnvironmentVariableNames.ExtraSparkSubmitArgs,
$"--packages com.microsoft.hyperspace:{hyperspaceVersion}");
SparkFixture = new SparkFixture();
}
public DeltaFixture()
{
Version sparkVersion = SparkSettings.Version;
string deltaVersion = sparkVersion.Major switch
{
2 => "delta-core_2.11:0.6.1",
3 => "delta-core_2.12:0.7.0",
_ => throw new NotSupportedException($"Spark {sparkVersion} not supported.")
};
Environment.SetEnvironmentVariable(
SparkFixture.EnvironmentVariableNames.ExtraSparkSubmitArgs,
$"--packages io.delta:{deltaVersion} " +
"--conf spark.databricks.delta.snapshotPartitions=2 " +
"--conf spark.sql.sources.parallelPartitionDiscovery.parallelism=5");
SparkFixture = new SparkFixture();
}
}
public FunctionsTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public BasicStatistics(SparkFixture fixture) => _session = fixture.Spark;
public TriggerTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public SparkConfTests(SparkFixture fixture)
{
_fixture = fixture;
}
public HashingTFTests(SparkFixture fixture) : base(fixture)
{
_spark = fixture.Spark;
}
public BucketizerTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public FileSystemTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public MutualInformationAnalyzer(SparkFixture fixture) => _session = fixture.Spark;
protected FeatureBaseTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public SparkSessionExtensionsTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public IDFModelTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public ParamTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public DataFrameWriterTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public Examples(SparkFixture fixture, ITestOutputHelper helper)
{
_session = fixture.Spark;
_helper = helper;
}
public BroadcastTests(SparkFixture fixture)
{
_spark = fixture.Spark;
_df = _spark.CreateDataFrame(new[] { "hello", "world" });
}
public Word2VecTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public CountVectorizerTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public SparkSessionTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
public CompletenessConstraint(SparkFixture fixture) => _session = fixture.Spark;
public StopWordsRemoverTests(SparkFixture fixture) : base(fixture)
{
_spark = fixture.Spark;
}
public CatalogTests(SparkFixture fixture)
{
_spark = fixture.Spark;
}
