public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf) { JvmObjectReference jvmSparkContextReference = JvmBridge.CallConstructor("org.apache.spark.SparkContext", (conf as SparkConfIpcProxy).JvmSparkConfReference); JvmObjectReference jvmJavaContextReference = JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference }); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); return sparkContextProxy; }
public IDStreamProxy SocketTextStream(string hostname, int port, StorageLevelType storageLevelType) { JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "socketTextStream", hostname, port, jlevel).ToString()); return(new DStreamIpcProxy(jstream)); }
public void Persist(StorageLevelType storageLevelType) { var jstorageLevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( jvmDataFrameReference, "persist", new object[] { jstorageLevel }); }
public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf) { JvmObjectReference jvmSparkContextReference = JvmBridge.CallConstructor("org.apache.spark.SparkContext", (conf as SparkConfIpcProxy).JvmSparkConfReference); JvmObjectReference jvmJavaContextReference = JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference }); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); return(sparkContextProxy); }
public IDStreamProxy KafkaStream(Dictionary <string, int> topics, Dictionary <string, string> kafkaParams, StorageLevelType storageLevelType) { JvmObjectReference jtopics = SparkContextIpcProxy.GetJavaMap <string, int>(topics); JvmObjectReference jkafkaParams = SparkContextIpcProxy.GetJavaMap <string, string>(kafkaParams); JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { }); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString()); return(new DStreamIpcProxy(jstream)); }
public IDStreamProxy KafkaStream(IEnumerable <Tuple <string, int> > topics, IEnumerable <Tuple <string, string> > kafkaParams, StorageLevelType storageLevelType) { JvmObjectReference jtopics = JvmBridgeUtils.GetJavaMap <string, int>(topics); JvmObjectReference jkafkaParams = JvmBridgeUtils.GetJavaMap <string, string>(kafkaParams); JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); // KafkaUtilsPythonHelper: external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { }); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString()); return(new DStreamIpcProxy(jstream)); }
public IDStreamProxy Union(IDStreamProxy firstDStream, IDStreamProxy[] otherDStreams) { return(new DStreamIpcProxy( new JvmObjectReference( (string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "union", new object[] { (firstDStream as DStreamIpcProxy).javaDStreamReference, SparkContextIpcProxy.GetJavaList <JvmObjectReference>(otherDStreams.Select(x => (x as DStreamIpcProxy).javaDStreamReference)) } )))); }
public IDStreamProxy EventHubsUnionStream(Dictionary <string, string> eventHubsParams, StorageLevelType storageLevelType) { JvmObjectReference eventHubsParamsReference = JvmBridgeUtils.GetScalaMutableMap <string, string>(eventHubsParams); JvmObjectReference storageLevelTypeReference = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); return (new DStreamIpcProxy( new JvmObjectReference( SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod( "org.apache.spark.streaming.api.csharp.EventHubsUtils", "createUnionStream", new object[] { jvmJavaStreamingReference, eventHubsParamsReference, storageLevelTypeReference }) .ToString()))); }
public IDStreamProxy DirectKafkaStream(List <string> topics, Dictionary <string, string> kafkaParams, Dictionary <string, long> fromOffsets) { JvmObjectReference jtopics = SparkContextIpcProxy.GetJavaSet <string>(topics); JvmObjectReference jkafkaParams = SparkContextIpcProxy.GetJavaMap <string, string>(kafkaParams); var jTopicAndPartitions = fromOffsets.Select(x => new KeyValuePair <JvmObjectReference, long> ( SparkCLRIpcProxy.JvmBridge.CallConstructor("kafka.common.TopicAndPartition", new object[] { x.Key.Split(':')[0], int.Parse(x.Key.Split(':')[1]) }), x.Value ) ); JvmObjectReference jfromOffsets = SparkContextIpcProxy.GetJavaMap <JvmObjectReference, long>(jTopicAndPartitions); JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { }); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createDirectStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jfromOffsets }).ToString()); return(new DStreamIpcProxy(jstream)); }
public BroadcastIpcProxy(JvmObjectReference jvmBroadcastReference, SparkContextIpcProxy sparkContextIpcProxy) { this.jvmBroadcastReference = jvmBroadcastReference; this.sparkContextIpcProxy = sparkContextIpcProxy; }
public void SaveAsHadoopFile(string path, string outputFormatClass, string keyClass, string valueClass, IEnumerable <KeyValuePair <string, string> > conf, string compressionCodecClass) { var jconf = SparkContextIpcProxy.GetJavaMap <string, string>(conf); SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "saveAsHadoopFile", new object[] { jvmRddReference, false, path, outputFormatClass, keyClass, valueClass, null, null, jconf, compressionCodecClass }); }
public void SaveAsHadoopDataset(IEnumerable <KeyValuePair <string, string> > conf) { var jconf = SparkContextIpcProxy.GetJavaMap <string, string>(conf); SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "saveAsHadoopDataset", new object[] { jvmRddReference, false, jconf, null, null, false }); }
public IRDDProxy SampleByKey(bool withReplacement, Dictionary <string, double> fractions, long seed) { var jfractions = SparkContextIpcProxy.GetJavaMap(fractions) as JvmObjectReference; return(new RDDIpcProxy(new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "sampleByKey", new object[] { withReplacement, jfractions, seed })))); }