public IDStreamProxy SocketTextStream(string hostname, int port, StorageLevelType storageLevelType)
{
JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "socketTextStream", hostname, port, jlevel).ToString());
return(new DStreamIpcProxy(jstream));
}
public void Persist(StorageLevelType storageLevelType)
{
var jstorageLevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(
jvmDataFrameReference, "persist", new object[] { jstorageLevel });
}
public IDStreamProxy KafkaStream(Dictionary <string, int> topics, Dictionary <string, string> kafkaParams, StorageLevelType storageLevelType)
{
JvmObjectReference jtopics = SparkContextIpcProxy.GetJavaMap <string, int>(topics);
JvmObjectReference jkafkaParams = SparkContextIpcProxy.GetJavaMap <string, string>(kafkaParams);
JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { });
var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString());
return(new DStreamIpcProxy(jstream));
}
public IDStreamProxy KafkaStream(IEnumerable <Tuple <string, int> > topics, IEnumerable <Tuple <string, string> > kafkaParams, StorageLevelType storageLevelType)
{
JvmObjectReference jtopics = JvmBridgeUtils.GetJavaMap <string, int>(topics);
JvmObjectReference jkafkaParams = JvmBridgeUtils.GetJavaMap <string, string>(kafkaParams);
JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
// KafkaUtilsPythonHelper: external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { });
var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString());
return(new DStreamIpcProxy(jstream));
}
public IDStreamProxy EventHubsUnionStream(Dictionary <string, string> eventHubsParams, StorageLevelType storageLevelType)
{
JvmObjectReference eventHubsParamsReference = JvmBridgeUtils.GetScalaMutableMap <string, string>(eventHubsParams);
JvmObjectReference storageLevelTypeReference = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
return
(new DStreamIpcProxy(
new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod(
"org.apache.spark.streaming.api.csharp.EventHubsUtils", "createUnionStream",
new object[] { jvmJavaStreamingReference, eventHubsParamsReference, storageLevelTypeReference })
.ToString())));
}
