public void Persist(StorageLevelType storageLevelType)
{
var jstorageLevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(
jvmDataFrameReference, "persist", new object[] { jstorageLevel });
}
public IDStreamProxy SocketTextStream(string hostname, int port, StorageLevelType storageLevelType)
{
JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "socketTextStream", hostname, port, jlevel).ToString());
return(new DStreamIpcProxy(jstream));
}
public static JvmObjectReference GetJavaStorageLevel(StorageLevelType storageLevelType)
{
return(new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.java.StorageLevels", "create",
new object[]
{
StorageLevel.storageLevel[storageLevelType].useDisk,
StorageLevel.storageLevel[storageLevelType].useMemory,
StorageLevel.storageLevel[storageLevelType].useOffHeap,
StorageLevel.storageLevel[storageLevelType].deserialized,
StorageLevel.storageLevel[storageLevelType].replication
}).ToString()));
}
/// <summary>
/// Create an input stream that pulls messages from a Kafka Broker.
/// </summary>
/// <param name="zkQuorum">Zookeeper quorum (hostname:port,hostname:port,..).</param>
/// <param name="groupId">The group id for this consumer.</param>
/// <param name="topics">Dict of (topic_name -> numPartitions) to consume. Each partition is consumed in its own thread.</param>
/// <param name="kafkaParams">Additional params for Kafka</param>
/// <param name="storageLevelType">RDD storage level.</param>
/// <returns>A DStream object</returns>
public static DStream<KeyValuePair<byte[], byte[]>> CreateStream(StreamingContext ssc, string zkQuorum, string groupId, Dictionary<string, int> topics, Dictionary<string, string> kafkaParams, StorageLevelType storageLevelType)
{
if (kafkaParams == null)
kafkaParams = new Dictionary<string, string>();
if (!string.IsNullOrEmpty(zkQuorum))
kafkaParams["zookeeper.connect"] = zkQuorum;
if (groupId != null)
kafkaParams["group.id"] = groupId;
if (kafkaParams.ContainsKey("zookeeper.connection.timeout.ms"))
kafkaParams["zookeeper.connection.timeout.ms"] = "10000";
return new DStream<KeyValuePair<byte[], byte[]>>(ssc.streamingContextProxy.KafkaStream(topics, kafkaParams, storageLevelType), ssc);
}
/// <summary>
/// Create an input stream that pulls messages from a Kafka Broker.
/// </summary>
/// <param name="zkQuorum">Zookeeper quorum (hostname:port,hostname:port,..).</param>
/// <param name="groupId">The group id for this consumer.</param>
/// <param name="topics">Dict of (topic_name -> numPartitions) to consume. Each partition is consumed in its own thread.</param>
/// <param name="kafkaParams">Additional params for Kafka</param>
/// <param name="storageLevelType">RDD storage level.</param>
/// <returns>A DStream object</returns>
public DStream <KeyValuePair <byte[], byte[]> > KafkaStream(string zkQuorum, string groupId, Dictionary <string, int> topics, Dictionary <string, string> kafkaParams, StorageLevelType storageLevelType)
{
if (kafkaParams == null)
{
kafkaParams = new Dictionary <string, string>();
}
if (!string.IsNullOrEmpty(zkQuorum))
{
kafkaParams["zookeeper.connect"] = zkQuorum;
}
if (groupId != null)
{
kafkaParams["group.id"] = groupId;
}
if (kafkaParams.ContainsKey("zookeeper.connection.timeout.ms"))
{
kafkaParams["zookeeper.connection.timeout.ms"] = "10000";
}
return(new DStream <KeyValuePair <byte[], byte[]> >(this.streamingContextProxy.KafkaStream(topics, kafkaParams, storageLevelType), this));
}
/// <summary>
/// Create an input from TCP source hostname:port. Data is received using
/// a TCP socket and receive byte is interpreted as UTF8 encoded ``\\n`` delimited
/// lines.
/// </summary>
/// <param name="hostname">Hostname to connect to for receiving data</param>
/// <param name="port">Port to connect to for receiving data</param>
/// <param name="storageLevelType">Storage level to use for storing the received objects</param>
/// <returns></returns>
public DStream <string> SocketTextStream(string hostname, int port, StorageLevelType storageLevelType = StorageLevelType.MEMORY_AND_DISK_SER_2)
{
return(new DStream <string>(streamingContextProxy.SocketTextStream(hostname, port, storageLevelType), this, SerializedMode.String));
}
public void Persist(StorageLevelType storageLevelType)
{
Validate();
}
public IDStreamProxy KafkaStream(Dictionary <string, int> topics, Dictionary <string, string> kafkaParams, StorageLevelType storageLevelType)
{
JvmObjectReference jtopics = JvmBridgeUtils.GetJavaMap <string, int>(topics);
JvmObjectReference jkafkaParams = JvmBridgeUtils.GetJavaMap <string, string>(kafkaParams);
JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
// KafkaUtilsPythonHelper: external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { });
var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString());
return(new DStreamIpcProxy(jstream));
}
/// <summary>
/// Persist the RDDs of this DStream with the given storage level
/// </summary>
/// <param name="storageLevelType"></param>
/// <returns></returns>
public DStream <T> Persist(StorageLevelType storageLevelType)
{
isCached = true;
DStreamProxy.Persist(storageLevelType);
return(this);
}
public void Persist(StorageLevelType storageLevelType)
{
var jstorageLevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "persist", new object[] { jstorageLevel });
}
public IDStreamProxy EventHubsUnionStream(Dictionary<string, string> eventHubsParams, StorageLevelType storageLevelType)
{
JvmObjectReference eventHubsParamsReference = JvmBridgeUtils.GetScalaMutableMap<string, string>(eventHubsParams);
JvmObjectReference storageLevelTypeReference = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
return
new DStreamIpcProxy(
new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod(
"org.apache.spark.streaming.api.csharp.EventHubsUtils", "createUnionStream",
new object[] { jvmJavaStreamingReference, eventHubsParamsReference, storageLevelTypeReference })
.ToString()));
}
public void Persist(StorageLevelType storageLevelType)
{
Validate();
}
public void Persist(StorageLevelType storageLevelType)
{
}
public IDStreamProxy EventHubsUnionStream(IEnumerable <Tuple <string, string> > eventHubsParams, StorageLevelType storageLevelType)
{
throw new NotImplementedException();
}
/// <summary>
/// Create an input stream that pulls messages from a Kafka Broker.
/// </summary>
/// <param name="ssc">Spark Streaming Context</param>
/// <param name="zkQuorum">Zookeeper quorum (hostname:port,hostname:port,..).</param>
/// <param name="groupId">The group id for this consumer.</param>
/// <param name="topics">Dict of (topic_name -> numPartitions) to consume. Each partition is consumed in its own thread.</param>
/// <param name="kafkaParams">Additional params for Kafka</param>
/// <param name="storageLevelType">RDD storage level.</param>
/// <returns>A DStream object</returns>
public static DStream <Tuple <byte[], byte[]> > CreateStream(StreamingContext ssc, string zkQuorum, string groupId, IEnumerable <Tuple <string, int> > topics, IEnumerable <Tuple <string, string> > kafkaParams, StorageLevelType storageLevelType)
{
if (kafkaParams == null)
{
kafkaParams = new List <Tuple <string, string> >();
}
var kafkaParamsMap = kafkaParams.ToDictionary(x => x.Item1, x => x.Item2);
if (!string.IsNullOrEmpty(zkQuorum))
{
kafkaParamsMap["zookeeper.connect"] = zkQuorum;
}
if (groupId != null)
{
kafkaParamsMap["group.id"] = groupId;
}
if (kafkaParamsMap.ContainsKey("zookeeper.connection.timeout.ms"))
{
kafkaParamsMap["zookeeper.connection.timeout.ms"] = "10000";
}
return(new DStream <Tuple <byte[], byte[]> >(ssc.streamingContextProxy.KafkaStream(topics, kafkaParamsMap.Select(x => Tuple.Create(x.Key, x.Value)), storageLevelType), ssc));
}
public void Persist(StorageLevelType storageLevelType)
{
}
public IDStreamProxy EventHubsUnionStream(Dictionary<string, string> eventHubsParams, StorageLevelType storageLevelType)
{
throw new NotImplementedException();
}
/// <summary>
/// Create a unioned EventHubs stream that receives data from Microsoft Azure Eventhubs
/// The unioned stream will receive message from all partitions of the EventHubs
/// </summary>
/// <param name="ssc">Streaming context</param>
/// <param name="eventhubsParams"> Parameters for EventHubs.
/// Required parameters are:
/// "eventhubs.policyname": EventHubs policy name
/// "eventhubs.policykey": EventHubs policy key
/// "eventhubs.namespace": EventHubs namespace
/// "eventhubs.name": EventHubs name
/// "eventhubs.partition.count": Number of partitions
/// "eventhubs.checkpoint.dir": checkpoint directory on HDFS
///
/// Optional parameters are:
/// "eventhubs.consumergroup": EventHubs consumer group name, default to "\$default"
/// "eventhubs.filter.offset": Starting offset of EventHubs, default to "-1"
/// "eventhubs.filter.enqueuetime": Unix time, millisecond since epoch, default to "0"
/// "eventhubs.default.credits": default AMQP credits, default to -1 (which is 1024)
/// "eventhubs.checkpoint.interval": checkpoint interval in second, default to 10
/// </param>
/// <param name="storageLevelType">Storage level, by default it is MEMORY_ONLY</param>
/// <returns>DStream with byte[] representing events from EventHub</returns>
public static DStream <byte[]> CreateUnionStream(StreamingContext ssc, Dictionary <string, string> eventhubsParams, StorageLevelType storageLevelType = StorageLevelType.MEMORY_ONLY)
{
return(new DStream <byte[]>(ssc.streamingContextProxy.EventHubsUnionStream(eventhubsParams, storageLevelType), ssc, SerializedMode.None));
}
public static JvmObjectReference GetJavaStorageLevel(StorageLevelType storageLevelType)
{
return new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.java.StorageLevels", "create",
new object[]
{
StorageLevel.storageLevel[storageLevelType].useDisk,
StorageLevel.storageLevel[storageLevelType].useMemory,
StorageLevel.storageLevel[storageLevelType].useOffHeap,
StorageLevel.storageLevel[storageLevelType].deserialized,
StorageLevel.storageLevel[storageLevelType].replication
}).ToString());
}
public IDStreamProxy SocketTextStream(string hostname, int port, StorageLevelType storageLevelType)
{
JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "socketTextStream", hostname, port, jlevel).ToString());
return new DStreamIpcProxy(jstream);
}
public IDStreamProxy KafkaStream(Dictionary<string, int> topics, Dictionary<string, string> kafkaParams, StorageLevelType storageLevelType)
{
JvmObjectReference jtopics = SparkContextIpcProxy.GetJavaMap<string, int>(topics);
JvmObjectReference jkafkaParams = SparkContextIpcProxy.GetJavaMap<string, string>(kafkaParams);
JvmObjectReference jlevel = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
JvmObjectReference jhelper = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.kafka.KafkaUtilsPythonHelper", new object[] { });
var jstream = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jhelper, "createStream", new object[] { jvmJavaStreamingReference, jkafkaParams, jtopics, jlevel }).ToString());
return new DStreamIpcProxy(jstream);
}
public IDStreamProxy EventHubsUnionStream(Dictionary <string, string> eventHubsParams, StorageLevelType storageLevelType)
{
throw new NotImplementedException();
}
public IDStreamProxy EventHubsUnionStream(Dictionary <string, string> eventHubsParams, StorageLevelType storageLevelType)
{
JvmObjectReference eventHubsParamsReference = JvmBridgeUtils.GetScalaMutableMap <string, string>(eventHubsParams);
JvmObjectReference storageLevelTypeReference = SparkContextIpcProxy.GetJavaStorageLevel(storageLevelType);
return
(new DStreamIpcProxy(
new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod(
"org.apache.spark.streaming.api.csharp.EventHubsUtils", "createUnionStream",
new object[] { jvmJavaStreamingReference, eventHubsParamsReference, storageLevelTypeReference })
.ToString())));
}
/// <summary>
/// Create a unioned EventHubs stream that receives data from Microsoft Azure Eventhubs
/// The unioned stream will receive message from all partitions of the EventHubs
/// </summary>
/// <param name="ssc">Streaming context</param>
/// <param name="eventhubsParams"> Parameters for EventHubs.
/// Required parameters are:
/// "eventhubs.policyname": EventHubs policy name
/// "eventhubs.policykey": EventHubs policy key
/// "eventhubs.namespace": EventHubs namespace
/// "eventhubs.name": EventHubs name
/// "eventhubs.partition.count": Number of partitions
/// "eventhubs.checkpoint.dir": checkpoint directory on HDFS
///
/// Optional parameters are:
/// "eventhubs.consumergroup": EventHubs consumer group name, default to "\$default"
/// "eventhubs.filter.offset": Starting offset of EventHubs, default to "-1"
/// "eventhubs.filter.enqueuetime": Unix time, millisecond since epoch, default to "0"
/// "eventhubs.default.credits": default AMQP credits, default to -1 (which is 1024)
/// "eventhubs.checkpoint.interval": checkpoint interval in second, default to 10
/// </param>
/// <param name="storageLevelType">Storage level, by default it is MEMORY_ONLY</param>
/// <returns>DStream with byte[] representing events from EventHub</returns>
public static DStream<byte[]> CreateUnionStream(StreamingContext ssc, Dictionary<string, string> eventhubsParams, StorageLevelType storageLevelType = StorageLevelType.MEMORY_ONLY)
{
return new DStream<byte[]>(ssc.streamingContextProxy.EventHubsUnionStream(eventhubsParams, storageLevelType), ssc, SerializedMode.None);
}
