internal static int RunJob(IRDDProxy rdd)
{
var mockRdd = (rdd as MockRddProxy);
IEnumerable <byte[]> result = mockRdd.pickle ? mockRdd.result.Cast <byte[]>() :
mockRdd.result.Select(x =>
{
var ms = new MemoryStream();
formatter.Serialize(ms, x);
return(ms.ToArray());
});
var listener = SocketFactory.CreateSocket();
listener.Listen();
Task.Run(() =>
{
using (var socket = listener.Accept())
using (var ns = socket.GetStream())
{
foreach (var item in result)
{
SerDe.Write(ns, item.Length);
SerDe.Write(ns, item);
}
ns.Flush();
}
});
return((listener.LocalEndPoint as IPEndPoint).Port);
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prefvJavaRddReference, byte[] command, Dictionary <string, string> environmentVariables, List <string> cSharpIncludes, bool preservePartitioning, List <Broadcast> broadcastVariables, List <byte[]> accumulator)
{
IEnumerable <dynamic> input = (prefvJavaRddReference as MockRddProxy).result ??
(new string[] {
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The dog lazy"
}).AsEnumerable().Cast <dynamic>();
using (MemoryStream s = new MemoryStream(command))
{
string deserializerMode = SerDe.ReadString(s);
string serializerMode = SerDe.ReadString(s);
var func = (Func <int, IEnumerable <dynamic>, IEnumerable <dynamic> >)formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
IEnumerable <dynamic> output = func(default(int), input);
// number 8 indicates shuffling scenario's leading 8-byte hash code of each data row which should be filtered
if (output.FirstOrDefault() is byte[] && (output.First() as byte[]).Length == 8)
{
output = output.Where(e => (e as byte[]).Length != 8).Select(e => formatter.Deserialize(new MemoryStream(e as byte[])));
}
return(new MockRddProxy(output));
}
}
internal static int RunJob(IRDDProxy rdd)
{
var mockRdd = (rdd as MockRddProxy);
IEnumerable <byte[]> result = mockRdd.pickle ? mockRdd.result.Cast <byte[]>() :
mockRdd.result.Select(x =>
{
var ms = new MemoryStream();
formatter.Serialize(ms, x);
return(ms.ToArray());
});
TcpListener listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
Task.Run(() =>
{
using (Socket socket = listener.AcceptSocket())
using (Stream ns = new NetworkStream(socket))
{
foreach (var item in result)
{
SerDe.Write(ns, item.Length);
SerDe.Write(ns, item);
}
}
});
return((listener.LocalEndpoint as IPEndPoint).Port);
}
public IRDDProxy Union(IRDDProxy javaRddReferenceOther)
{
var union = new MockRddProxy(new object[] { this, javaRddReferenceOther });
if (result != null)
union.result = result.Concat((javaRddReferenceOther as MockRddProxy).result);
return union;
}
public IDataFrameProxy CreateDataFrame(IRDDProxy rddProxy, IStructTypeProxy structTypeProxy)
{
var rdd = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "byteArrayRDDToAnyArrayRDD",
new object[] { (rddProxy as RDDIpcProxy).JvmRddReference }).ToString());
return(new DataFrameIpcProxy(
new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmSparkSessionReference, "applySchemaToPythonRDD",
new object[] { rdd, (structTypeProxy as StructTypeIpcProxy).JvmStructTypeReference }).ToString()), sqlContextProxy));
}
public IDataFrameProxy CreateDataFrame(IRDDProxy rddProxy, IStructTypeProxy structTypeProxy)
{
var rdd = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "byteArrayRDDToAnyArrayRDD",
new object[] { (rddProxy as RDDIpcProxy).JvmRddReference }).ToString());
return new DataFrameIpcProxy(
new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmSqlContextReference, "applySchemaToPythonRDD",
new object[] { rdd, (structTypeProxy as StructTypeIpcProxy).JvmStructTypeReference }).ToString()), this);
}
public IRDDProxy Union(IRDDProxy javaRddReferenceOther)
{
var union = new MockRddProxy(new object[] { this, javaRddReferenceOther });
if (result != null)
{
union.result = result.Concat((javaRddReferenceOther as MockRddProxy).result);
}
return(union);
}
/// <summary>
/// Create a PairwiseRDD.
/// </summary>
/// <param name="jvmReferenceOfByteArrayRdd"></param>
/// <param name="numPartitions"></param>
/// <param name="partitionFuncId">Global unique id of partitioner which is used for comparison PythonPartitioners in JVM.</param>
/// <returns></returns>
public IRDDProxy CreatePairwiseRDD(IRDDProxy jvmReferenceOfByteArrayRdd, int numPartitions, long partitionFuncId)
{
var rdd = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((jvmReferenceOfByteArrayRdd as RDDIpcProxy).JvmRddReference, "rdd"));
var pairwiseRdd = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.python.PairwiseRDD", rdd);
var pairRddJvmReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(pairwiseRdd, "asJavaPairRDD", new object[] { }).ToString());
var jpartitionerJavaReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.python.PythonPartitioner", new object[] { numPartitions, partitionFuncId });
var partitionedPairRddJvmReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(pairRddJvmReference, "partitionBy", new object[] { jpartitionerJavaReference }).ToString());
var jvmRddReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "valueOfPair", new object[] { partitionedPairRddJvmReference }).ToString());
//var jvmRddReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(partitionedRddJvmReference, "rdd", new object[] { }).ToString());
return(new RDDIpcProxy(jvmRddReference));
}
public IDStreamProxy CreateConstantInputDStream(IRDDProxy rddProxy)
{
var rddReference =
new JvmObjectReference(
(string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(((RDDIpcProxy)rddProxy).JvmRddReference, "rdd"));
var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor(
"org.apache.spark.streaming.api.csharp.CSharpConstantInputDStream", jvmStreamingContextReference, rddReference);
var javaDStreamReference =
new JvmObjectReference((String)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream"));
return(new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference));
}
public void TestDStreamTransform_Moq()
{
// Arrange
var mockDStreamProxy = new Mock <IDStreamProxy>();
_mockStreamingContextProxy.Setup(m => m.TextFileStream(It.Is <string>(d => d == Path.GetTempPath()))).Returns(mockDStreamProxy.Object);
mockDStreamProxy.Setup(m => m.CallForeachRDD(It.IsAny <byte[]>(), It.IsAny <string>())).Callback <byte[], string>(
(func, deserializer) =>
{
Action <double, RDD <dynamic> > f = (Action <double, RDD <dynamic> >) new BinaryFormatter().Deserialize(new MemoryStream(func));
f(DateTime.UtcNow.Ticks, new RDD <dynamic>(_mockRddProxy.Object, new SparkContext("", "")));
});
IRDDProxy functionedRddProxy = null;
mockDStreamProxy.Setup(m => m.AsJavaDStream()).Returns(mockDStreamProxy.Object);
_mockSparkCLRProxy.Setup(m => m.StreamingContextProxy.CreateCSharpDStream(It.IsAny <IDStreamProxy>(), It.IsAny <byte[]>(), It.IsAny <string>()))
.Returns <IDStreamProxy, byte[], string>((jdstream, func, deserializer) =>
{
Func <double, RDD <dynamic>, RDD <dynamic> > f = (Func <double, RDD <dynamic>, RDD <dynamic> >) new BinaryFormatter().Deserialize(new MemoryStream(func));
RDD <dynamic> rdd = f(DateTime.UtcNow.Ticks,
new RDD <dynamic>(functionedRddProxy ?? _mockRddProxy.Object, new SparkContext("", "")));
functionedRddProxy = rdd.RddProxy;
return(mockDStreamProxy.Object);
});
// Act
var lines = _streamingContext.TextFileStream(Path.GetTempPath());
var words = lines.FlatMap(l => l.Split(' '));
var pairs = words.Map(w => new Tuple <string, int>(w, 1));
var wordCounts = pairs.ReduceByKey((x, y) => x + y);
// Assert
wordCounts.ForeachRDD((time, rdd) =>
{
var taken = rdd.Collect();
Assert.AreEqual(taken.Length, 9);
foreach (object record in taken)
{
Tuple <string, int> countByWord = (Tuple <string, int>)record;
Assert.AreEqual(countByWord.Item2, countByWord.Item1 == "The" || countByWord.Item1 == "dog" || countByWord.Item1 == "lazy" ? 23 : 22);
}
});
// Use Verify to verify if a method to mock was invoked
mockDStreamProxy.Verify(m => m.CallForeachRDD(It.IsAny <byte[]>(), It.IsAny <string>()));
}
/// <summary>
/// Returns all of Rows in this DataFrame
/// </summary>
public IEnumerable <Row> Collect()
{
if (rowSchema == null)
{
rowSchema = RowSchema.ParseRowSchemaFromJson(Schema.ToJson());
}
IRDDProxy rddProxy = dataFrameProxy.JavaToCSharp();
RDD <Row> rdd = new RDD <Row>(rddProxy, sparkContext, SerializedMode.Row);
int port = rddProxy.CollectAndServe();
foreach (var item in rdd.Collect(port))
{
yield return(new RowImpl(item, rowSchema));
}
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prevJvmRddReference, byte[] command, Dictionary <string, string> environmentVariables, List <string> pythonIncludes, bool preservesPartitioning, List <Broadcast> broadcastVariables, List <byte[]> accumulator)
{
var hashTableReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("java.util.Hashtable", new object[] { });
var arrayListReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("java.util.ArrayList", new object[] { });
var jbroadcastVariables = JvmBridgeUtils.GetJavaList <JvmObjectReference>(jvmBroadcastReferences);
var rdd = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((prevJvmRddReference as RDDIpcProxy).JvmRddReference, "rdd"));
var csRdd = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.csharp.CSharpRDD",
new object[]
{
rdd, command, hashTableReference, arrayListReference, preservesPartitioning,
SparkCLREnvironment.ConfigurationService.GetCSharpWorkerExePath(),
"1.0",
jbroadcastVariables, jvmAccumulatorReference
});
return(new RDDIpcProxy(new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(csRdd, "asJavaRDD"))));
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prefvJavaRddReference, byte[] command, Dictionary<string, string> environmentVariables, List<string> cSharpIncludes, bool preservePartitioning, List<Broadcast> broadcastVariables, List<byte[]> accumulator)
{
IEnumerable<dynamic> input = (prefvJavaRddReference as MockRddProxy).result ??
(new string[] {
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The dog lazy"
}).AsEnumerable().Cast<dynamic>();
using (MemoryStream s = new MemoryStream(command))
{
int rddId = SerDe.ReadInt(s);
int stageId = SerDe.ReadInt(s);
int partitionId = SerDe.ReadInt(s);
string deserializerMode = SerDe.ReadString(s);
string serializerMode = SerDe.ReadString(s);
CSharpWorkerFunc workerFunc = (CSharpWorkerFunc)formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
var func = workerFunc.Func;
IEnumerable<dynamic> output = func(default(int), input);
// number 8 indicates shuffling scenario's leading 8-byte hash code of each data row which should be filtered
if (output.FirstOrDefault() is byte[] && (output.First() as byte[]).Length == 8)
{
output = output.Where(e => (e as byte[]).Length != 8).Select(e => formatter.Deserialize(new MemoryStream(e as byte[])));
}
return new MockRddProxy(output);
}
}
public IRDDProxy CreatePairwiseRDD(IRDDProxy javaReferenceInByteArrayRdd, int numPartitions)
{
return(javaReferenceInByteArrayRdd);
}
public IRDDProxy Cartesian(IRDDProxy other)
{
Validate();
return(this);
}
public int RunJob(IRDDProxy rdd, IEnumerable <int> partitions)
{
return(RunJob(rdd));
}
public IDataFrameProxy CreateDataFrame(IRDDProxy rddProxy, IStructTypeProxy structTypeProxy)
{
throw new NotImplementedException();
}
public MockDStreamProxy(IRDDProxy rddProxy)
{
this.rddProxy = rddProxy;
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prevJvmRddReference, byte[] command, Dictionary<string, string> environmentVariables, List<string> pythonIncludes, bool preservesPartitioning, List<Broadcast> broadcastVariables, List<byte[]> accumulator)
{
var hashTableReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("java.util.Hashtable", new object[] { });
var arrayListReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("java.util.ArrayList", new object[] { });
var jbroadcastVariables = JvmBridgeUtils.GetJavaList<JvmObjectReference>(jvmBroadcastReferences);
var rdd = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((prevJvmRddReference as RDDIpcProxy).JvmRddReference, "rdd"));
var csRdd = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.csharp.CSharpRDD",
new object[]
{
rdd, command, hashTableReference, arrayListReference, preservesPartitioning,
SparkCLREnvironment.ConfigurationService.GetCSharpWorkerExePath(),
"1.0",
jbroadcastVariables, jvmAccumulatorReference
});
return new RDDIpcProxy(new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(csRdd, "asJavaRDD")));
}
public int RunJob(IRDDProxy rdd, IEnumerable<int> partitions)
{
var jpartitions = JvmBridgeUtils.GetJavaList<int>(partitions);
return int.Parse(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "runJob", new object[] { jvmSparkContextReference, (rdd as RDDIpcProxy).JvmRddReference, jpartitions }).ToString());
}
public IDStreamProxy CreateConstantInputDStream(IRDDProxy rddProxy)
{
return(new MockDStreamProxy());
}
internal static int RunJob(IRDDProxy rdd)
{
var mockRdd = (rdd as MockRddProxy);
IEnumerable<byte[]> result = mockRdd.pickle ? mockRdd.result.Cast<byte[]>() :
mockRdd.result.Select(x =>
{
var ms = new MemoryStream();
formatter.Serialize(ms, x);
return ms.ToArray();
});
var listener = SocketFactory.CreateSocket();
listener.Listen();
Task.Run(() =>
{
using (var socket = listener.Accept())
using (var ns = socket.GetStream())
{
foreach (var item in result)
{
SerDe.Write(ns, item.Length);
SerDe.Write(ns, item);
}
ns.Flush();
}
});
return (listener.LocalEndPoint as IPEndPoint).Port;
}
public IRDDProxy Zip(IRDDProxy other)
{
var rdd = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "rdd"));
return(new RDDIpcProxy(new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "zip", new object[] { (other as RDDIpcProxy).jvmRddReference }))));
}
public IRDDProxy Union(IRDDProxy javaRddReferenceOther)
{
var jref = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "union", new object[] { (javaRddReferenceOther as RDDIpcProxy).jvmRddReference }).ToString());
return(new RDDIpcProxy(jref));
}
public IRDDProxy Cartesian(IRDDProxy other)
{
return(new RDDIpcProxy(new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "cartesian", (other as RDDIpcProxy).jvmRddReference))));
}
public IDStreamProxy CreateConstantInputDStream(IRDDProxy rddProxy)
{
var rddReference =
new JvmObjectReference(
(string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(((RDDIpcProxy)rddProxy).JvmRddReference, "rdd"));
var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor(
"org.apache.spark.streaming.api.csharp.CSharpConstantInputDStream", jvmStreamingContextReference, rddReference);
var javaDStreamReference =
new JvmObjectReference((String)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream"));
return new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference);
}
public IRDDProxy Zip(IRDDProxy other)
{
Validate();
return(this);
}
public IRDDProxy Cartesian(IRDDProxy other)
{
return this;
}
public IRDDProxy Cartesian(IRDDProxy other)
{
return(this);
}
public MockDStreamProxy(IRDDProxy rddProxy)
{
this.rddProxy = rddProxy;
}
public IRDDProxy Zip(IRDDProxy other)
{
return(this);
}
public IRDDProxy CreatePairwiseRDD(IRDDProxy jvmReferenceOfByteArrayRdd, int numPartitions)
{
var rdd = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((jvmReferenceOfByteArrayRdd as RDDIpcProxy).JvmRddReference, "rdd"));
var pairwiseRdd = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.python.PairwiseRDD", rdd);
var pairRddJvmReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(pairwiseRdd, "asJavaPairRDD", new object[] { }).ToString());
var jpartitionerJavaReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.python.PythonPartitioner", new object[] { numPartitions, (long)0 });
var partitionedPairRddJvmReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(pairRddJvmReference, "partitionBy", new object[] { jpartitionerJavaReference }).ToString());
var jvmRddReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "valueOfPair", new object[] { partitionedPairRddJvmReference }).ToString());
//var jvmRddReference = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(partitionedRddJvmReference, "rdd", new object[] { }).ToString());
return new RDDIpcProxy(jvmRddReference);
}
internal static int RunJob(IRDDProxy rdd)
{
var mockRdd = (rdd as MockRddProxy);
IEnumerable<byte[]> result = mockRdd.pickle ? mockRdd.result.Cast<byte[]>() :
mockRdd.result.Select(x =>
{
var ms = new MemoryStream();
formatter.Serialize(ms, x);
return ms.ToArray();
});
TcpListener listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
Task.Run(() =>
{
using (Socket socket = listener.AcceptSocket())
using (Stream ns = new NetworkStream(socket))
{
foreach (var item in result)
{
SerDe.Write(ns, item.Length);
SerDe.Write(ns, item);
}
}
});
return (listener.LocalEndpoint as IPEndPoint).Port;
}
public IDataFrameProxy CreateDataFrame(IRDDProxy rddProxy, IStructTypeProxy structTypeProxy)
{
throw new NotImplementedException();
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prefvJavaRddReference, byte[] command, Dictionary<string, string> environmentVariables, List<string> pythonIncludes, bool preservePartitioning, List<Broadcast<dynamic>> broadcastVariables, List<byte[]> accumulator)
{
throw new NotImplementedException();
}
public IRDDProxy Zip(IRDDProxy other)
{
return this;
}
public IRDDProxy CreatePairwiseRDD(IRDDProxy javaReferenceInByteArrayRdd, int numPartitions, long partitionFuncId)
{
return javaReferenceInByteArrayRdd;
}
public IRDDProxy Cartesian(IRDDProxy other)
{
Validate();
return this;
}
public int RunJob(IRDDProxy rdd, IEnumerable<int> partitions, bool allowLocal)
{
return RunJob(rdd);
}
public int RunJob(IRDDProxy rdd, IEnumerable<int> partitions)
{
return RunJob(rdd);
}
public IDStreamProxy CreateConstantInputDStream(IRDDProxy rddProxy)
{
return new MockDStreamProxy();
}
public IRDDProxy Intersection(IRDDProxy other)
{
return new RDDIpcProxy(new JvmObjectReference((string) SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "intersection", new object[] { (other as RDDIpcProxy).jvmRddReference })));
}
public IRDDProxy Union(IRDDProxy javaRddReferenceOther)
{
var jref = new JvmObjectReference(SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "union", new object[] { (javaRddReferenceOther as RDDIpcProxy).jvmRddReference }).ToString());
return new RDDIpcProxy(jref);
}
public IRDDProxy Zip(IRDDProxy other)
{
Validate();
return this;
}
public IRDDProxy Zip(IRDDProxy other)
{
var rdd = new JvmObjectReference((string) SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "rdd"));
return new RDDIpcProxy(new JvmObjectReference((string) SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "zip", new object[] { (other as RDDIpcProxy).jvmRddReference })));
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prefvJavaRddReference, byte[] command, Dictionary <string, string> environmentVariables, List <string> pythonIncludes, bool preservePartitioning, List <Broadcast <dynamic> > broadcastVariables, List <byte[]> accumulator)
{
throw new NotImplementedException();
}
public IRDDProxy Cartesian(IRDDProxy other)
{
return new RDDIpcProxy(new JvmObjectReference((string) SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmRddReference, "cartesian", (other as RDDIpcProxy).jvmRddReference)));
}
public int RunJob(IRDDProxy rdd, IEnumerable <int> partitions, bool allowLocal)
{
return(RunJob(rdd));
}
public int RunJob(IRDDProxy rdd, IEnumerable <int> partitions)
{
var jpartitions = JvmBridgeUtils.GetJavaList <int>(partitions);
return(int.Parse(SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.api.python.PythonRDD", "runJob", new object[] { jvmSparkContextReference, (rdd as RDDIpcProxy).JvmRddReference, jpartitions }).ToString()));
}
