public IDStreamProxy SocketTextStream(string hostname, int port, StorageLevelType storageLevelType) { JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "socketTextStream", hostname, port, jlevel).ToString()); return(new DStreamIpcProxy(jstream)); }
public void Persist(StorageLevelType storageLevelType) { var jstorageLevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( jvmDataFrameReference, "persist", new object[] { jstorageLevel }); }
public IDStreamProxy KafkaStream(Dictionary <string, int> topics, Dictionary <string, string> kafkaParams, StorageLevelType storageLevelType) { JvmObjectReference jtopics = SparkContextIpcProxy.GetJavaMap <string, int>(topics); JvmObjectReference jkafkaParams = SparkContextIpcProxy.GetJavaMap <string, string>(kafkaParams); JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { }); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString()); return(new DStreamIpcProxy(jstream)); }
public IDStreamProxy KafkaStream(IEnumerable <Tuple <string, int> > topics, IEnumerable <Tuple <string, string> > kafkaParams, StorageLevelType storageLevelType) { JvmObjectReference jtopics = JvmBridgeUtils.GetJavaMap <string, int>(topics); JvmObjectReference jkafkaParams = JvmBridgeUtils.GetJavaMap <string, string>(kafkaParams); JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); // KafkaUtilsPythonHelper: external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { }); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString()); return(new DStreamIpcProxy(jstream)); }
public IDStreamProxy EventHubsUnionStream(Dictionary <string, string> eventHubsParams, StorageLevelType storageLevelType) { JvmObjectReference eventHubsParamsReference = JvmBridgeUtils.GetScalaMutableMap <string, string>(eventHubsParams); JvmObjectReference storageLevelTypeReference = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); return (new DStreamIpcProxy( new JvmObjectReference( SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod( "org.apache.spark.streaming.api.csharp.EventHubsUtils", "createUnionStream", new object[] { jvmJavaStreamingReference, eventHubsParamsReference, storageLevelTypeReference }) .ToString()))); }