/// <summary>
/// Call https://github.com/apache/spark/blob/branch-1.4/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala, except(other: DataFrame): DataFrame
/// </summary>
/// <param name="otherScalaDataFrameReference"></param>
/// <returns></returns>
public IDataFrameProxy Subtract(IDataFrameProxy otherScalaDataFrameReference)
{
return
(new DataFrameIpcProxy(new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(
jvmDataFrameReference, "except",
new object[] { (otherScalaDataFrameReference as DataFrameIpcProxy).jvmDataFrameReference }).ToString()), sqlContextProxy));
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string joinColumnName)
{
return
(new DataFrameIpcProxy(new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDataFrameReference, "join", new object[]
{
(otherScalaDataFrameReference as DataFrameIpcProxy).jvmDataFrameReference,
joinColumnName
}).ToString()
), sqlContextProxy));
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string[] joinColumnNames)
{
var stringSequenceReference = new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "toSeq", new object[] { joinColumnNames }).ToString());
return
(new DataFrameIpcProxy(new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDataFrameReference, "join", new object[]
{
(otherScalaDataFrameReference as DataFrameIpcProxy).jvmDataFrameReference,
stringSequenceReference
}).ToString()
), sqlContextProxy));
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string[] joinColumnNames)
{
throw new NotSupportedException("Not supported in 1.4.1");
//TODO - uncomment this in 1.5
//var stringSequenceReference = new JvmObjectReference(
// SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "toSeq", new object[] { joinColumnNames }).ToString());
//return
// new JvmObjectReference(
// SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(scalaDataFrameReference, "join", new object[]
// {
// otherScalaDataFrameReference,
// stringSequenceReference
// }).ToString()
// );
}
internal DataFrame(IDataFrameProxy dataFrameProxy, SparkContext sparkContext)
{
this.dataFrameProxy = dataFrameProxy;
this.sparkContext = sparkContext;
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string[] joinColumnNames)
{
throw new NotImplementedException();
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, IColumnProxy scalaColumnReference, string joinType)
{
throw new NotImplementedException();
}
public void RegisterDataFrameAsTable(IDataFrameProxy dataFrameProxy, string tableName)
{
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(
jvmSqlContextReference, "registerDataFrameAsTable",
new object[] { (dataFrameProxy as DataFrameIpcProxy).JvmDataFrameReference, tableName });
}
public void RegisterDataFrameAsTable(IDataFrameProxy dataFrameProxy, string tableName)
{
throw new NotImplementedException();
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string joinColumnName)
{
return new MockDataFrameProxy(new object[] { otherScalaDataFrameReference, joinColumnName }, SqlContextProxy);
}
public void RegisterDataFrameAsTable(IDataFrameProxy dataFrameProxy, string tableName)
{
throw new NotImplementedException();
}
public IDataFrameProxy UnionAll(IDataFrameProxy other)
{
throw new NotImplementedException();
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string[] joinColumnNames)
{
throw new NotSupportedException("Not supported in 1.4.1");
//TODO - uncomment this in 1.5
//var stringSequenceReference = new JvmObjectReference(
// SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "toSeq", new object[] { joinColumnNames }).ToString());
//return
// new JvmObjectReference(
// SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(scalaDataFrameReference, "join", new object[]
// {
// otherScalaDataFrameReference,
// stringSequenceReference
// }).ToString()
// );
}
public IDataFrameProxy UnionAll(IDataFrameProxy other)
{
throw new NotImplementedException();
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, IColumnProxy scalaColumnReference, string joinType)
{
return
new DataFrameIpcProxy(new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(
jvmDataFrameReference, "join",
new object[]
{
(otherScalaDataFrameReference as DataFrameIpcProxy).jvmDataFrameReference,
(scalaColumnReference as ColumnIpcProxy).ScalaColumnReference,
joinType
}).ToString()), sqlContextProxy);
}
public IDataFrameProxy Subtract(IDataFrameProxy other)
{
throw new NotImplementedException();
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, IColumnProxy scalaColumnReference, string joinType)
{
throw new NotImplementedException();
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string[] joinColumnNames)
{
throw new NotImplementedException();
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, string[] joinColumnNames)
{
return(new MockDataFrameProxy(new object[] { otherScalaDataFrameReference, joinColumnNames }, SqlContextProxy));
}
public void RegisterDataFrameAsTable(IDataFrameProxy dataFrameProxy, string tableName)
{
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(
jvmSqlContextReference, "registerDataFrameAsTable",
new object[] { (dataFrameProxy as DataFrameIpcProxy).JvmDataFrameReference, tableName });
}
public IDataFrameProxy Join(IDataFrameProxy otherScalaDataFrameReference, IColumnProxy scalaColumnReference, string joinType)
{
return(new MockDataFrameProxy(new object[] { otherScalaDataFrameReference, scalaColumnReference, joinType }, SqlContextProxy));
}
public IDataFrameProxy UnionAll(IDataFrameProxy otherScalaDataFrameReference)
{
return
new DataFrameIpcProxy(new JvmObjectReference(
SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(
jvmDataFrameReference, "unionAll",
new object[] { (otherScalaDataFrameReference as DataFrameIpcProxy).jvmDataFrameReference }).ToString()), sqlContextProxy);
}
public IDataFrameProxy Subtract(IDataFrameProxy other)
{
throw new NotImplementedException();
}
internal DataFrame(IDataFrameProxy dataFrameProxy, SparkContext sparkContext)
{
this.dataFrameProxy = dataFrameProxy;
this.sparkContext = sparkContext;
}
