public void TestSparkContextConstructor()
{
var sparkContext = new SparkContext("masterUrl", "appName");
Assert.IsNotNull((sparkContext.SparkContextProxy as MockSparkContextProxy).mockSparkContextReference);
var paramValuesToConstructor = (sparkContext.SparkContextProxy as MockSparkContextProxy).mockSparkContextReference as object[];
Assert.AreEqual("masterUrl", (paramValuesToConstructor[0] as MockSparkConfProxy).stringConfDictionary["mockmaster"]);
Assert.AreEqual("appName", (paramValuesToConstructor[0] as MockSparkConfProxy).stringConfDictionary["mockappName"]);
Assert.AreEqual(sparkContext, SparkContext.GetActiveSparkContext());
sparkContext = new SparkContext("masterUrl", "appName", "sparkhome");
Assert.IsNotNull((sparkContext.SparkContextProxy as MockSparkContextProxy).mockSparkContextReference);
paramValuesToConstructor = (sparkContext.SparkContextProxy as MockSparkContextProxy).mockSparkContextReference as object[];
Assert.AreEqual("masterUrl", (paramValuesToConstructor[0] as MockSparkConfProxy).stringConfDictionary["mockmaster"]);
Assert.AreEqual("appName", (paramValuesToConstructor[0] as MockSparkConfProxy).stringConfDictionary["mockappName"]);
Assert.AreEqual("sparkhome", (paramValuesToConstructor[0] as MockSparkConfProxy).stringConfDictionary["mockhome"]);
Assert.AreEqual(sparkContext, SparkContext.GetActiveSparkContext());
sparkContext = new SparkContext(null);
Assert.IsNotNull((sparkContext.SparkContextProxy as MockSparkContextProxy).mockSparkContextReference);
paramValuesToConstructor = (sparkContext.SparkContextProxy as MockSparkContextProxy).mockSparkContextReference as object[];
Assert.IsNotNull(paramValuesToConstructor[0]); //because SparkContext constructor create default sparkConf
Assert.AreEqual(sparkContext, SparkContext.GetActiveSparkContext());
}
internal RDD <T> Execute(double t)
{
var sc = SparkContext.GetActiveSparkContext();
int[] array = new int[numPartitions];
var initialRdd = sc.Parallelize(array.AsEnumerable(), numPartitions);
return(initialRdd.MapPartitionsWithIndex <T>(new CSharpInputDStreamMapPartitionWithIndexHelper <T>(t, func).Execute, true));
}
/// <summary>
/// Converts this strongly typed collection of data to generic Dataframe. In contrast to the
/// strongly typed objects that Dataset operations work on, a Dataframe returns generic[[Row]]
/// objects that allow fields to be accessed by ordinal or name.
/// </summary>
/// <returns>DataFrame created from Dataset</returns>
public DataFrame ToDF()
{
return(dataFrame ?? (dataFrame = new DataFrame(datasetProxy.ToDF(), SparkContext.GetActiveSparkContext())));
}
public DataFrame CreateExternalTable(string tableName, string path, string source)
{
return(new DataFrame(
new DataFrameIpcProxy(
new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmCatalogReference, "createExternalTable",
new object[] { tableName, path, source }).ToString()), sqlContextProxy), SparkContext.GetActiveSparkContext()));
}