public IDStreamProxy KafkaStream(Dictionary <string, int> topics, Dictionary <string, string> kafkaParams, StorageLevelType storageLevelType) { JvmObjectReference jtopics = SparkContextIpcProxy.GetJavaMap <string, int>(topics); JvmObjectReference jkafkaParams = SparkContextIpcProxy.GetJavaMap <string, string>(kafkaParams); JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType); JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { }); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString()); return(new DStreamIpcProxy(jstream)); }
public IDStreamProxy DirectKafkaStream(List <string> topics, Dictionary <string, string> kafkaParams, Dictionary <string, long> fromOffsets) { JvmObjectReference jtopics = SparkContextIpcProxy.GetJavaSet <string>(topics); JvmObjectReference jkafkaParams = SparkContextIpcProxy.GetJavaMap <string, string>(kafkaParams); var jTopicAndPartitions = fromOffsets.Select(x => new KeyValuePair <JvmObjectReference, long> ( SparkCLRIpcProxy.JvmBridge.CallConstructor("kafka.common.TopicAndPartition", new object[] { x.Key.Split(':')[0], int.Parse(x.Key.Split(':')[1]) }), x.Value ) ); JvmObjectReference jfromOffsets = SparkContextIpcProxy.GetJavaMap <JvmObjectReference, long>(jTopicAndPartitions); JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { }); var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createDirectStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jfromOffsets }).ToString()); return(new DStreamIpcProxy(jstream)); }
public void SaveAsHadoopDataset(IEnumerable <KeyValuePair <string, string> > conf) { var jconf = SparkContextIpcProxy.GetJavaMap <string, string>(conf); SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "saveAsHadoopDataset", new object[] { jvmRddReference, false, jconf, null, null, false }); }
public void SaveAsHadoopFile(string path, string outputFormatClass, string keyClass, string valueClass, IEnumerable <KeyValuePair <string, string> > conf, string compressionCodecClass) { var jconf = SparkContextIpcProxy.GetJavaMap <string, string>(conf); SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "saveAsHadoopFile", new object[] { jvmRddReference, false, path, outputFormatClass, keyClass, valueClass, null, null, jconf, compressionCodecClass }); }
public IRDDProxy SampleByKey(bool withReplacement, Dictionary <string, double> fractions, long seed) { var jfractions = SparkContextIpcProxy.GetJavaMap(fractions) as JvmObjectReference; return(new RDDIpcProxy(new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "sampleByKey", new object[] { withReplacement, jfractions, seed })))); }