/// <summary>
/// Sinks data until receives null then closes adapter
/// </summary>
/// <param name="data"></param>
public override void Write(DataFrame data)
{
if (data == null)
{
_exitEvent.Set();
}
}
/// <summary>
/// Write a frame to the adapter
/// </summary>
/// <param name="frame">The frame</param>
public override void Write(DataFrame frame)
{
if (frame != null)
{
_listener.Write(frame.ToArray(), _ep);
}
}
/// <summary>
/// Creates a new <see cref="DataCell"/> from specified parameters.
/// </summary>
/// <param name="parent">The reference to parent <see cref="DataFrame"/> of this <see cref="DataCell"/>.</param>
/// <param name="configurationCell">The <see cref="ConfigurationCell"/> associated with this <see cref="DataCell"/>.</param>
/// <param name="addEmptyValues">If <c>true</c>, adds empty values for each defined configuration cell definition.</param>
public DataCell(DataFrame parent, ConfigurationCell configurationCell, bool addEmptyValues)
: this(parent, configurationCell)
{
if (addEmptyValues)
{
int x;
// Define needed phasor values
for (x = 0; x < configurationCell.PhasorDefinitions.Count; x++)
{
PhasorValues.Add(new PhasorValue(this, configurationCell.PhasorDefinitions[x]));
}
// Define a frequency and df/dt
FrequencyValue = new FrequencyValue(this, configurationCell.FrequencyDefinition);
// Define any analog values
for (x = 0; x < configurationCell.AnalogDefinitions.Count; x++)
{
AnalogValues.Add(new AnalogValue(this, configurationCell.AnalogDefinitions[x]));
}
// Define any digital values
for (x = 0; x < configurationCell.DigitalDefinitions.Count; x++)
{
DigitalValues.Add(new DigitalValue(this, configurationCell.DigitalDefinitions[x]));
}
}
}
public static IEnumerable<DataFrame> ParseFrames(IEnumerable<DataFrame> frames, string selectionPath, ScriptContainer container, string classname)
{
BasePipelineNode input;
ParseWithPipelineNode output;
IEnumerable<DataFrame> ret = new DataFrame[0];
NetGraph graph = BuildGraph(container, classname, selectionPath, out input, out output);
try
{
foreach (DataFrame frame in frames)
{
input.Input(frame);
}
input.Shutdown(null);
output.EventFlag.WaitOne(500);
ret = output.Frames;
}
finally
{
((IDisposable)graph).Dispose();
}
return ret;
}
public void ReceiveAsync(Action<string> callback, DataFrame frame = null)
{
var buffer = new byte[256];
if (frame == null)
frame = new DataFrame();
Socket.AsyncReceive(buffer, frame, (sizeOfReceivedData, df) =>
{
var dataframe = (DataFrame)df;
if (sizeOfReceivedData > 0)
{
dataframe.Append(buffer);
if (dataframe.IsComplete)
{
var data = dataframe.ToString();
callback(data);
}
else // end is not is this buffer
{
ReceiveAsync(callback, dataframe); // continue to read
}
}
});
}
/// <summary>
/// Constructs a new incoming packet received by a TCP connector.
/// Use this class to receive data from a server.
/// This is an incoming message that will remain in the client's thread pool
/// as a job for the thread pool workers.
/// </summary>
/// <param name="connector">The TCP Connector where this message belongs to.</param>
/// <param name="data">DataFrame class.</param>
/// <param name="previousPacket">The previous incoming message of the TCP Connector.</param>
public IncomingTCPClientPacket(TCPConnector connector, DataFrame data, IncomingTCPClientPacket previousPacket)
{
fConnector = connector;
fData = data;
fPreviousPacket = previousPacket;
ThreadPool.QueueUserWorkItem(new WaitCallback(ReadData));
}
/// <summary>
/// Called when a new frame arrives (just forwards)
/// </summary>
/// <param name="frame"></param>
protected override void OnInput(DataFrame frame)
{
if (PacketDelayMs > 0)
{
Thread.Sleep(PacketDelayMs);
}
WriteOutput(frame);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="frame">The data frame to edit</param>
/// <param name="selectPath">Path to select a node to edit</param>
/// <param name="sender">The sending node</param>
/// <param name="color">The colour to show in an edit window</param>
/// <param name="tag">The textual tag to show in an edit window</param>
public EditPacketEventArgs(DataFrame frame, string selectPath, BasePipelineNode sender, ColorValue color, string tag)
: base()
{
Frame = frame;
SelectPath = selectPath;
Sender = sender;
Color = color;
Tag = tag;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="tag">The log tag</param>
/// <param name="netId">The log network ID</param>
/// <param name="frame">The log frame</param>
/// <param name="color">The log colour</param>
/// <param name="networkDescription">The log description</param>
public LogPacketEventArgs(string tag, Guid netId, DataFrame frame, ColorValue color, string networkDescription)
{
Tag = tag;
NetId = netId;
Frame = frame;
Color = color;
NetworkDescription = networkDescription;
Timestamp = DateTime.Now;
}
public void can_get_index_of_non_existant_column()
{
var dataFrame = new DataFrame(
new IntColumn("Column1"),
new StringColumn("Column2")
);
Assert.Equal(-1, dataFrame.GetColumnIndex("non-existant-column"));
}
public MDNS_Packet(DataFrame DataFrame)
{
byte[] Payload = DataFrame.Payload;
this.TransactionId = BitConverter.ToUInt16(Payload, 0);
this.Response = (MDNS_Response)BitConverter.ToUInt16(Payload, 2);
this.Questions = BitConverter.ToUInt16(Payload, 4);
this.AnswerRRs = BitConverter.ToUInt16(Payload, 6);
this.AuthorityRRs = BitConverter.ToUInt16(Payload, 8);
this.AdditionalRRs = BitConverter.ToUInt16(Payload, 10);
}
/// <summary>
/// Convert a packet to a hex string format
/// </summary>
/// <param name="p">The packet to convert</param>
/// <returns>The converted string</returns>
public static string ConvertBinaryPacketToString(DataFrame p)
{
using (TextWriter writer = new StringWriter())
{
writer.WriteLine(GeneralUtils.BuildHexDump(16, p.ToArray()));
writer.WriteLine();
return writer.ToString();
}
}
/// <summary>
/// Private Constructor
/// </summary>
/// <param name="tag"></param>
/// <param name="netid"></param>
/// <param name="uuid"></param>
/// <param name="network"></param>
/// <param name="frame"></param>
/// <param name="color"></param>
/// <param name="timestamp"></param>
public LogPacket(string tag, Guid netid, Guid uuid, string network, DataFrame frame, ColorValue color, DateTime timestamp)
{
Tag = tag;
NetId = netid;
Uuid = uuid;
Network = network;
Frame = frame;
Color = color;
Timestamp = timestamp;
}
public void can_get_columns()
{
var dataFrame = new DataFrame(
new IntColumn("Column1"),
new StringColumn("Column2")
);
var columnNames = new string[] { "Column1", "Column2" };
Assert.Equal(columnNames, dataFrame.GetColumnNames());
}
public void can_get_column_index()
{
var dataFrame = new DataFrame(
new IntColumn("Column1"),
new StringColumn("Column2")
);
Assert.Equal(0, dataFrame.GetColumnIndex("Column1"));
Assert.Equal(1, dataFrame.GetColumnIndex("Column2"));
}
DataFrame XorFrame(DataFrame frame, byte xorValue)
{
byte[] data = frame.ToArray();
for(int i = 0; i < data.Length; ++i)
{
data[i] = (byte)(data[i] ^ xorValue);
}
return new DataFrame(data);
}
public static int Main ()
{
DataFrame df1 = new DataFrame ();
DataFrame df2 = new DataFrame ();
if (df1 != null)
{
return 1;
}
return 0;
}
/// <summary>
/// Override function called when a packet is input
/// </summary>
/// <param name="frame">The input frame</param>
public override void Input(DataFrame frame)
{
EnsureThreadRunning();
try
{
_input.Enqueue(frame);
}
catch (InvalidOperationException)
{ }
catch (OperationCanceledException)
{ }
}
static void Main(string[] args)
{
//
// Create a simple data frame.
//
var maxRange = 14;
var dataFrame = new DataFrame(
new IntColumn("index", Enumerable.Range(0, maxRange)),
new DoubleColumn("Sin", Enumerable.Range(0, maxRange).Select(i => Math.Sin(i))),
new DoubleColumn("Cos", Enumerable.Range(0, maxRange).Select(i => Math.Cos(i)))
);
Console.WriteLine(dataFrame.ToString());
}
private void Receive(UserContext context, DataFrame frame)
{
var bytes = frame.Payload;
Assert.AreEqual(MessageSize, bytes.Length);
for (var i = 0; i < bytes.Length; i++)
{
var value = bytes[i];
var expectedValue = (byte)(i % 256);
Assert.AreEqual(value, expectedValue);
}
countdownEvent.Signal();
}
public string Receive(DataFrame frame = null)
{
if (frame == null)
frame = new DataFrame();
var buffer = new byte[256];
int sizeOfReceivedData = Socket.Receive(buffer);
frame.Append(buffer);
if (frame.IsComplete)
return frame.ToString();
else
return Receive(frame);
}
private static SimpleHyperCube convert(DataFrame dataFrame)
{
dynamic df = dataFrame;
object[] colnames = Enumerable.ToArray(SymbolicExpressionExtension.AsVector(df.names));
object[] tmp = Enumerable.ToArray(SymbolicExpressionExtension.AsVector(df.name));
var varnames = Array.ConvertAll(tmp, x => (string)x);
SimpleHyperCube s = new SimpleHyperCube(varnames);
var rows = dataFrame.GetRows();
foreach (var vn in varnames)
{
dynamic row = rows.First(x => ((string)((dynamic)x).name == vn));
s.SetMinMaxValue(vn, row.min, row.max, row.value);
}
return s;
}
/// <summary>
/// Constructs a new incoming packet for an existing TCP client connection.
///
/// Use this class to receive data from a client.
/// This is an incoming message that will remain in the servers thread pool
/// as a job for the thread pool workers.
/// </summary>
/// <param name="client">The client where this message belongs to.</param>
/// <param name="data">DataFrame class.</param>
/// <param name="previousPacket">The previous incoming message of the client.</param>
public IncomingTCPClientConnectionPacket(TCPClientConnection client, DataFrame data, IncomingTCPClientConnectionPacket previousPacket)
{
try
{
fClient = client;
fData = data;
fPreviousPacket = previousPacket;
}
catch (Exception ex)
{
return;
}
ThreadPool.QueueUserWorkItem(new WaitCallback(ReadData));
}
/// <summary>
/// Method called when a new frame arraives
/// </summary>
/// <param name="frame">The frame</param>
protected override void OnInput(DataFrame frame)
{
DataNode[] nodes = frame.SelectNodes(SelectionPath);
foreach (DataNode node in nodes)
{
try
{
MemoryStream stm = new MemoryStream(node.ToArray());
DataReader reader = new DataReader(stm);
string name = node.Name;
DataKey parentKey = node.Parent;
node.RemoveNode();
while (stm.Position < stm.Length)
{
DynamicStreamDataKey2 key = new DynamicStreamDataKey2(name, Container, Graph.Logger, State);
reader.ByteCount = 0;
key.FromReader(reader);
// The reader clearly didn't care
if (reader.ByteCount == 0)
{
break;
}
parentKey.AddSubNode(key);
frame.Current = key;
}
}
catch (EndOfStreamException)
{
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
LogException(ex);
}
}
WriteOutput(frame);
}
/// <summary>
/// Extended match syntax, can use details from a graph and node
/// </summary>
/// <param name="frame">The data frame to select off</param>
/// <param name="globalMeta">The global meta</param>
/// <param name="meta">Meta</param>
/// <param name="node">The current node</param>
/// <param name="properties">Property bag</param>
/// <param name="uuid">The uuid for private names</param>
/// <returns>True if the filter matches</returns>
public bool IsMatch(DataFrame frame, MetaDictionary meta, MetaDictionary globalMeta, PropertyBag properties,
Guid uuid, BasePipelineNode node)
{
bool match = false;
DataNode[] nodes = GeneralUtils.SelectNodes(Path, frame, meta, globalMeta, properties, uuid, node);
match = OnMatch(nodes);
if (Invert)
{
return !match;
}
else
{
return match;
}
}
/// <summary>
///
/// </summary>
/// <param name="frame"></param>
protected override void OnInput(DataFrame frame)
{
bool match = false;
if (Filters != null)
{
match = Filters.IsMatch(frame, Graph.Meta, Graph.GlobalMeta, Graph.ConnectionProperties, Graph.Uuid, this);
}
if (match)
{
WriteOutput(frame, _pathName);
}
else
{
WriteOutputExclude(frame, _pathName);
}
}
/// <summary>
/// Set the frame to edit
/// </summary>
/// <param name="frame">The data frame</param>
/// <param name="selector">Root selection path</param>
/// <param name="color">The colour to display the frame in (if applicable)</param>
public void SetFrame(DataFrame frame, string selector, Color color)
{
DataNode node = null;
DataNode curr = frame.Current;
_frame = frame;
if (!String.IsNullOrWhiteSpace(selector))
{
node = _frame.SelectSingleNode(selector);
}
if (node == null)
{
node = _frame.Root;
}
dataNodeEditorControl.SetNode(node, curr, color, ReadOnly);
}
public void Test_RegressionWith_SimpleKnnModel()
{
// Given
var randomizer = new Random(55);
var baseDataFrame = TestDataBuilder.BuildRandomAbstractNumericDataFrameWithRedundantAttrs(randomizer: randomizer);
var queryDataFrame = new DataFrame(new DataTable("some data")
{
Columns =
{
new DataColumn("F1", typeof(double)),
new DataColumn("F2", typeof(double)),
new DataColumn("F3", typeof(double)),
new DataColumn("F4", typeof(double)),
new DataColumn("F5", typeof(double))
},
Rows =
{
new object[] { 10, 1, 1, 4, 5 },
new object[] { 4, 2, 1, 9, 10},
new object[] { 2, 1, 1, 3, 7},
}
});
var expectedValues = Enumerable.Range(0, queryDataFrame.RowCount)
.Select(
rowIdx =>
TestDataBuilder.CalcualteLinearlyDependentFeatureValue(queryDataFrame.GetNumericRowVector(rowIdx))).ToList();
var modelBuilder = new SimpleKnnModelBuilder<double>();
var modelParams = new KnnAdditionalParams(4, true);
var weightingFunction = new GaussianFunction(0.3);
var predictor = new SimpleKnnRegressor(new EuclideanDistanceMeasure(), new MinMaxNormalizer(), weightingFunction.GetValue, normalizeNumericValues: true);
var errorMeasure = new MeanSquareError();
// When
var model = modelBuilder.BuildModel(baseDataFrame, "F6", modelParams);
var results = predictor.Predict(queryDataFrame, model, "F6");
// Then
var mse = errorMeasure.CalculateError(Vector<double>.Build.DenseOfEnumerable(expectedValues), Vector<double>.Build.DenseOfEnumerable(results));
Assert.IsTrue(mse < 0.55);
}
public void can_get_rows()
{
var dataFrame = new DataFrame(
new IntColumn("Column1", new int[] { 1, 2 }),
new StringColumn("Column2", new string[] { "A", "B" })
);
var rows = dataFrame.GetRows().ToArray();
/*todo:
Assert.Equal(2, rows.Length);
Assert.Equal(1, rows[0].ByColumn("Column1").As<int>());
Assert.Equal(1, rows[0].ByColumn(0).As<int>());
Assert.Equal("A", rows[0].ByColumn("Column2").As<string>());
Assert.Equal("A", rows[0].ByColumn(1).As<string>());
Assert.Equal(2, rows[1].ByColumn("Column1").As<int>());
Assert.Equal(2, rows[1].ByColumn(0).As<int>());
Assert.Equal("B", rows[1].ByColumn("Column2").As<string>());
Assert.Equal("B", rows[1].ByColumn(1).As<string>());
*/
}
/// <summary>
/// Read a frame from the UDP client
/// </summary>
/// <returns>A data frame, null on error</returns>
public override DataFrame Read()
{
DataFrame frame = null;
IPEndPoint ep = null;
try
{
byte[] data = _client.Receive(ref ep);
if (data != null)
{
frame = new DataFrame(data);
}
}
catch(SocketException)
{
}
return frame;
}
internal static void VerifyColumnTypes(DataFrame df, bool testArrowStringColumn = false)
{
foreach (DataFrameColumn column in df.Columns)
{
Type dataType = column.DataType;
if (dataType == typeof(bool))
{
Assert.IsType <BooleanDataFrameColumn>(column);
}
else if (dataType == typeof(decimal))
{
Assert.IsType <DecimalDataFrameColumn>(column);
}
else if (dataType == typeof(byte))
{
Assert.IsType <ByteDataFrameColumn>(column);
}
else if (dataType == typeof(char))
{
Assert.IsType <CharDataFrameColumn>(column);
}
else if (dataType == typeof(double))
{
Assert.IsType <DoubleDataFrameColumn>(column);
}
else if (dataType == typeof(float))
{
Assert.IsType <SingleDataFrameColumn>(column);
}
else if (dataType == typeof(int))
{
Assert.IsType <Int32DataFrameColumn>(column);
}
else if (dataType == typeof(long))
{
Assert.IsType <Int64DataFrameColumn>(column);
}
else if (dataType == typeof(sbyte))
{
Assert.IsType <SByteDataFrameColumn>(column);
}
else if (dataType == typeof(short))
{
Assert.IsType <Int16DataFrameColumn>(column);
}
else if (dataType == typeof(uint))
{
Assert.IsType <UInt32DataFrameColumn>(column);
}
else if (dataType == typeof(ulong))
{
Assert.IsType <UInt64DataFrameColumn>(column);
}
else if (dataType == typeof(ushort))
{
Assert.IsType <UInt16DataFrameColumn>(column);
}
else if (dataType == typeof(string))
{
if (!testArrowStringColumn)
{
Assert.IsType <StringDataFrameColumn>(column);
}
else
{
Assert.IsType <ArrowStringDataFrameColumn>(column);
}
}
else if (dataType == typeof(DateTime))
{
Assert.IsType <PrimitiveDataFrameColumn <DateTime> >(column);
}
else
{
throw new NotImplementedException("Unit test has to be updated");
}
}
}
public void TestComputations()
{
DataFrame df = MakeDataFrameWithAllColumnTypes(10);
df["Int"][0] = -10;
Assert.Equal(-10, df["Int"][0]);
df["Int"].Abs();
Assert.Equal(10, df["Int"][0]);
Assert.Throws <NotSupportedException>(() => df["Byte"].All());
Assert.Throws <NotSupportedException>(() => df["Byte"].Any());
Assert.Throws <NotSupportedException>(() => df["Char"].All());
Assert.Throws <NotSupportedException>(() => df["Char"].Any());
Assert.Throws <NotSupportedException>(() => df["Decimal"].All());
Assert.Throws <NotSupportedException>(() => df["Decimal"].Any());
Assert.Throws <NotSupportedException>(() => df["Double"].All());
Assert.Throws <NotSupportedException>(() => df["Double"].Any());
Assert.Throws <NotSupportedException>(() => df["Float"].All());
Assert.Throws <NotSupportedException>(() => df["Float"].Any());
Assert.Throws <NotSupportedException>(() => df["Int"].All());
Assert.Throws <NotSupportedException>(() => df["Int"].Any());
Assert.Throws <NotSupportedException>(() => df["Long"].All());
Assert.Throws <NotSupportedException>(() => df["Long"].Any());
Assert.Throws <NotSupportedException>(() => df["Sbyte"].All());
Assert.Throws <NotSupportedException>(() => df["Sbyte"].Any());
Assert.Throws <NotSupportedException>(() => df["Short"].All());
Assert.Throws <NotSupportedException>(() => df["Short"].Any());
Assert.Throws <NotSupportedException>(() => df["Uint"].All());
Assert.Throws <NotSupportedException>(() => df["Uint"].Any());
Assert.Throws <NotSupportedException>(() => df["Ulong"].All());
Assert.Throws <NotSupportedException>(() => df["Ulong"].Any());
Assert.Throws <NotSupportedException>(() => df["Ushort"].All());
Assert.Throws <NotSupportedException>(() => df["Ushort"].Any());
bool any = df["Bool"].Any();
bool all = df["Bool"].All();
Assert.True(any);
Assert.False(all);
// Test the computation results
df["Double"][0] = 100.0;
df["Double"].CumulativeMax();
Assert.Equal(100.0, df["Double"][9]);
df["Float"][0] = -10.0f;
df["Float"].CumulativeMin();
Assert.Equal(-10.0f, df["Float"][9]);
df["Uint"].CumulativeProduct();
Assert.Equal((uint)0, df["Uint"][9]);
df["Ushort"].CumulativeSum();
Assert.Equal((ushort)40, df["Ushort"][9]);
Assert.Equal(100.0, df["Double"].Max());
Assert.Equal(-10.0f, df["Float"].Min());
Assert.Equal((uint)0, df["Uint"].Product());
Assert.Equal((ushort)140, df["Ushort"].Sum());
df["Double"][0] = 100.1;
Assert.Equal(100.1, df["Double"][0]);
df["Double"].Round();
Assert.Equal(100.0, df["Double"][0]);
// Test that none of the numeric column types throw
for (int i = 0; i < df.ColumnCount; i++)
{
BaseColumn column = df.Column(i);
if (column.DataType == typeof(bool))
{
Assert.Throws <NotSupportedException>(() => column.CumulativeMax());
Assert.Throws <NotSupportedException>(() => column.CumulativeMin());
Assert.Throws <NotSupportedException>(() => column.CumulativeProduct());
Assert.Throws <NotSupportedException>(() => column.CumulativeSum());
Assert.Throws <NotSupportedException>(() => column.Max());
Assert.Throws <NotSupportedException>(() => column.Min());
Assert.Throws <NotSupportedException>(() => column.Product());
Assert.Throws <NotSupportedException>(() => column.Sum());
continue;
}
else if (column.DataType == typeof(string))
{
Assert.Throws <NotImplementedException>(() => column.CumulativeMax());
Assert.Throws <NotImplementedException>(() => column.CumulativeMin());
Assert.Throws <NotImplementedException>(() => column.CumulativeProduct());
Assert.Throws <NotImplementedException>(() => column.CumulativeSum());
Assert.Throws <NotImplementedException>(() => column.Max());
Assert.Throws <NotImplementedException>(() => column.Min());
Assert.Throws <NotImplementedException>(() => column.Product());
Assert.Throws <NotImplementedException>(() => column.Sum());
continue;
}
column.CumulativeMax();
column.CumulativeMin();
column.CumulativeProduct();
column.CumulativeSum();
column.Max();
column.Min();
column.Product();
column.Sum();
}
}
public void TestDataFrameNaFunctionSignatures()
{
DataFrameNaFunctions dfNaFuncs = _df.Na();
var emptyColumn = new string[] { };
var validColumn = new string[] { "age" };
DataFrame df = dfNaFuncs.Drop("any");
df = dfNaFuncs.Drop("all");
df = dfNaFuncs.Drop(emptyColumn);
df = dfNaFuncs.Drop(validColumn);
df = dfNaFuncs.Drop("any", emptyColumn);
df = dfNaFuncs.Drop("all", validColumn);
df = dfNaFuncs.Drop(20);
df = dfNaFuncs.Drop(20, emptyColumn);
df = dfNaFuncs.Drop(20, validColumn);
df = dfNaFuncs.Fill(100L);
df = dfNaFuncs.Fill(100.0);
df = dfNaFuncs.Fill("hello");
df = dfNaFuncs.Fill(false);
df = dfNaFuncs.Fill(100L, emptyColumn);
df = dfNaFuncs.Fill(100L, validColumn);
df = dfNaFuncs.Fill(100.0, emptyColumn);
df = dfNaFuncs.Fill(100.0, validColumn);
df = dfNaFuncs.Fill("hello", emptyColumn);
df = dfNaFuncs.Fill("hello", validColumn);
df = dfNaFuncs.Fill(true, emptyColumn);
df = dfNaFuncs.Fill(true, validColumn);
df = dfNaFuncs.Fill(new Dictionary <string, int>()
{
{ "age", 10 }
});
df = dfNaFuncs.Fill(new Dictionary <string, long>()
{
{ "age", 10L }
});
df = dfNaFuncs.Fill(new Dictionary <string, double>()
{
{ "age", 10.0 }
});
df = dfNaFuncs.Fill(new Dictionary <string, string>()
{
{ "age", "name" }
});
df = dfNaFuncs.Fill(new Dictionary <string, bool>()
{
{ "age", false }
});
var doubleReplacement = new Dictionary <double, double>()
{
{ 1.0, 5.0 }
};
var boolReplacement = new Dictionary <bool, bool>()
{
{ true, false }
};
var stringReplacement = new Dictionary <string, string>()
{
{ "a", "b" }
};
df = dfNaFuncs.Replace("age", doubleReplacement);
df = dfNaFuncs.Replace("age", boolReplacement);
df = dfNaFuncs.Replace("age", stringReplacement);
df = dfNaFuncs.Replace(emptyColumn, doubleReplacement);
df = dfNaFuncs.Replace(validColumn, doubleReplacement);
df = dfNaFuncs.Replace(emptyColumn, boolReplacement);
df = dfNaFuncs.Replace(validColumn, boolReplacement);
df = dfNaFuncs.Replace(emptyColumn, stringReplacement);
df = dfNaFuncs.Replace(validColumn, stringReplacement);
}
public void TestSignatures()
{
using var tempDirectory = new TemporaryDirectory();
string path = Path.Combine(tempDirectory.Path, "delta-table");
DataFrame rangeRate = _spark.Range(15);
rangeRate.Write().Format("delta").Save(path);
DeltaTable table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(path));
table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(_spark, path));
Assert.IsType <bool>(DeltaTable.IsDeltaTable(_spark, path));
Assert.IsType <bool>(DeltaTable.IsDeltaTable(path));
Assert.IsType <DeltaTable>(table.As("oldTable"));
Assert.IsType <DeltaTable>(table.Alias("oldTable"));
Assert.IsType <DataFrame>(table.History());
Assert.IsType <DataFrame>(table.History(200));
Assert.IsType <DataFrame>(table.ToDF());
DataFrame newTable = _spark.Range(10, 15).As("newTable");
Assert.IsType <DeltaMergeBuilder>(
table.Merge(newTable, Functions.Exp("oldTable.id == newTable.id")));
DeltaMergeBuilder mergeBuilder = Assert.IsType <DeltaMergeBuilder>(
table.Merge(newTable, "oldTable.id == newTable.id"));
// Validate the MergeBuilder matched signatures.
Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched());
Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched("id = 5"));
DeltaMergeMatchedActionBuilder matchedActionBuilder =
Assert.IsType <DeltaMergeMatchedActionBuilder>(
mergeBuilder.WhenMatched(Functions.Expr("id = 5")));
Assert.IsType <DeltaMergeBuilder>(
matchedActionBuilder.Update(new Dictionary <string, Column>()));
Assert.IsType <DeltaMergeBuilder>(
matchedActionBuilder.UpdateExpr(new Dictionary <string, string>()));
Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.UpdateAll());
Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.Delete());
// Validate the MergeBuilder not-matched signatures.
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(mergeBuilder.WhenNotMatched());
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
mergeBuilder.WhenNotMatched("id = 5"));
DeltaMergeNotMatchedActionBuilder notMatchedActionBuilder =
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
mergeBuilder.WhenNotMatched(Functions.Expr("id = 5")));
Assert.IsType <DeltaMergeBuilder>(
notMatchedActionBuilder.Insert(new Dictionary <string, Column>()));
Assert.IsType <DeltaMergeBuilder>(
notMatchedActionBuilder.InsertExpr(new Dictionary <string, string>()));
Assert.IsType <DeltaMergeBuilder>(notMatchedActionBuilder.InsertAll());
// Update and UpdateExpr should return void.
table.Update(new Dictionary <string, Column>()
{
});
table.Update(Functions.Expr("id % 2 == 0"), new Dictionary <string, Column>()
{
});
table.UpdateExpr(new Dictionary <string, string>()
{
});
table.UpdateExpr("id % 2 == 1", new Dictionary <string, string>()
{
});
Assert.IsType <DataFrame>(table.Vacuum());
Assert.IsType <DataFrame>(table.Vacuum(168));
// Delete should return void.
table.Delete("id > 10");
table.Delete(Functions.Expr("id > 5"));
table.Delete();
// Load the table as a streaming source.
Assert.IsType <DataFrame>(_spark
.ReadStream()
.Format("delta")
.Option("path", path)
.Load());
Assert.IsType <DataFrame>(_spark.ReadStream().Format("delta").Load(path));
// Create Parquet data and convert it to DeltaTables.
string parquetIdentifier = $"parquet.`{path}`";
rangeRate.Write().Mode(SaveMode.Overwrite).Parquet(path);
Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(_spark, parquetIdentifier));
rangeRate
.Select(Functions.Col("id"), Functions.Expr($"(`id` + 1) AS `id_plus_one`"))
.Write()
.PartitionBy("id")
.Mode(SaveMode.Overwrite)
.Parquet(path);
Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(
_spark,
parquetIdentifier,
"id bigint"));
// TODO: Test with StructType partition schema once StructType is supported.
}
public void TestTutorialScenario()
{
using var tempDirectory = new TemporaryDirectory();
string path = Path.Combine(tempDirectory.Path, "delta-table");
// Write data to a Delta table.
DataFrame data = _spark.Range(0, 5);
data.Write().Format("delta").Save(path);
// Validate that data contains the the sequence [0 ... 4].
ValidateRangeDataFrame(Enumerable.Range(0, 5), data);
// Create a second iteration of the table.
data = _spark.Range(5, 10);
data.Write().Format("delta").Mode("overwrite").Save(path);
// Load the data into a DeltaTable object.
var deltaTable = DeltaTable.ForPath(path);
// Validate that deltaTable contains the the sequence [5 ... 9].
ValidateRangeDataFrame(Enumerable.Range(5, 5), deltaTable.ToDF());
// Update every even value by adding 100 to it.
deltaTable.Update(
condition: Functions.Expr("id % 2 == 0"),
set: new Dictionary <string, Column>()
{
{ "id", Functions.Expr("id + 100") }
});
// Validate that deltaTable contains the the data:
// +---+
// | id|
// +---+
// | 5|
// | 7|
// | 9|
// |106|
// |108|
// +---+
ValidateRangeDataFrame(
new List <int>()
{
5, 7, 9, 106, 108
},
deltaTable.ToDF());
// Delete every even value.
deltaTable.Delete(condition: Functions.Expr("id % 2 == 0"));
// Validate that deltaTable contains:
// +---+
// | id|
// +---+
// | 5|
// | 7|
// | 9|
// +---+
ValidateRangeDataFrame(new List <int>()
{
5, 7, 9
}, deltaTable.ToDF());
// Upsert (merge) new data.
DataFrame newData = _spark.Range(0, 20).As("newData").ToDF();
deltaTable.As("oldData")
.Merge(newData, "oldData.id = newData.id")
.WhenMatched()
.Update(
new Dictionary <string, Column>()
{
{ "id", Functions.Col("newData.id") }
})
.WhenNotMatched()
.InsertExpr(new Dictionary <string, string>()
{
{ "id", "newData.id" }
})
.Execute();
// Validate that the resulTable contains the the sequence [0 ... 19].
ValidateRangeDataFrame(Enumerable.Range(0, 20), deltaTable.ToDF());
}
internal DataStreamWriter(JvmObjectReference jvmObject, DataFrame df)
{
Reference = jvmObject;
_df = df;
}
private void OnSendToWebSocket(DataFrame data, HttpRequest request)
{
data.Send(request.Session);
}
public virtual object FrameDeserialize(DataFrame data, PipeStream stream)
{
return(stream.ReadString((int)data.Length));
}
public ActionResult ExecuteWS(HttpRequest request, DataFrame dataFrame)
{
JToken dataToken = (JToken)Newtonsoft.Json.JsonConvert.DeserializeObject((string)dataFrame.Body);
return(this.mActionFactory.ExecuteWithWS(request, this, dataToken));
}
// STATUS [ July 9, 2019 ] : this works
public string[] GetColumnNames(DataFrame df)
{
var columnHeaders = df.ColumnNames;
return(columnHeaders);
}
public void ARIM_Differencing_Test()
{
var dict = new Dictionary <string, List <object> >
{
{ "a", new List <object>()
{
1, 2, 3, 4, 5, 6
} },
{ "b", new List <object>()
{
1, 1, 2, 3, 5, 8
} },
{ "c", new List <object>()
{
1, 4, 9, 16, 25, 36
} },
};
//
var df = new DataFrame(dict);
//seasonal differencing period 1
var newDf = df.Diff(step: 1);
Assert.Equal(new List <object>()
{
DataFrame.NAN, 1, 1, 1, 1, 1
}, newDf["a"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, 0, 1, 1, 2, 3
}, newDf["b"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, 3, 5, 7, 9, 11
}, newDf["c"]);
//seasonal differencing period 2
newDf = df.Diff(step: 2);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, 2, 2, 2, 2
}, newDf["a"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, 1, 2, 3, 5
}, newDf["b"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, 8, 12, 16, 20
}, newDf["c"]);
//seasonal differencing period 3
newDf = df.Diff(step: 3);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 3, 3, 3
}, newDf["a"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 2, 4, 6
}, newDf["b"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 15, 21, 27
}, newDf["c"]);
}
public void ARIM_Diff_Recursive_Test()
{
var dict = new Dictionary <string, List <object> >
{
{ "a", new List <object>()
{
1, 2, 3, 4, 5, 6
} },
{ "b", new List <object>()
{
1, 1, 2, 3, 5, 8
} },
{ "c", new List <object>()
{
1, 4, 9, 16, 25, 36
} },
};
//
var df = new DataFrame(dict);
//seasonal differencing period 1
var newDf = df.Diff(1, type: DiffType.Recurrsive);
Assert.Equal(new List <object>()
{
DataFrame.NAN, 1, 1, 1, 1, 1
}, newDf["a"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, 0, 1, 1, 2, 3
}, newDf["b"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, 3, 5, 7, 9, 11
}, newDf["c"]);
//seasonal differencing period 2
newDf = df.Diff(step: 2, type: DiffType.Recurrsive);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, 0, 0, 0, 0
}, newDf["a"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, 1, 0, 1, 1
}, newDf["b"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, 2, 2, 2, 2
}, newDf["c"]);
//seasonal differencing period 3
newDf = df.Diff(step: 3, type: DiffType.Recurrsive);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 0, 0, 0
}, newDf["a"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, -1, 1, 0
}, newDf["b"]);
Assert.Equal(new List <object>()
{
DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 0, 0, 0
}, newDf["c"]);
}
/// <summary>
/// Executes the <see cref="Bucketizer"/> and transforms the DataFrame to include the new
/// column or columns with the bucketed data.
/// </summary>
/// <param name="source">The DataFrame to add the bucketed data to</param>
/// <returns>
/// <see cref="DataFrame"/> containing the original data and the new bucketed columns
/// </returns>
public DataFrame Transform(DataFrame source)
{
return(new DataFrame((JvmObjectReference)_jvmObject.Invoke("transform", source)));
}
public void SortBy_QuickSort_Test01()
{
var dict = new Dictionary <string, List <object> >
{
{ "col1", new List <object>()
{
1, 31, 41, 51, 61, 11, 21, 71, 81, 91
} },
{ "col2", new List <object>()
{
2, 32, 42, 52, 62, 12, 22, 72, 82, 92
} },
{ "col3", new List <object>()
{
3, 43, 33, 63, 53, 13, 23, 73, 83, 93
} },
{ "col4", new List <object>()
{
4, 54, 44, 34, 64, 14, 24, 74, 84, 94
} },
};
//
DataFrame.qsAlgo = true;
var df = new DataFrame(dict);
var dict1 = new Dictionary <string, List <object> >
{
{ "col1", new List <object>()
{
1, 11, 21, 31, 41, 51, 61, 71, 81, 91
} },
{ "col2", new List <object>()
{
2, 12, 22, 32, 42, 52, 62, 72, 82, 92
} },
{ "col3", new List <object>()
{
3, 13, 23, 43, 33, 63, 53, 73, 83, 93
} },
{ "col4", new List <object>()
{
4, 14, 24, 54, 44, 34, 64, 74, 84, 94
} },
};
var df1 = new DataFrame(dict1);
var result = df.SortBy(new string[] { "col1", "col2", "col3", "col4" });
var STR = result.ToStringBuilder();
var resultIndex = new object[] { 0, 5, 6, 1, 2, 3, 4, 7, 8, 9 };
Assert.Equal(result.Index, result.Index);
for (int i = 0; i < result.Values.Count; i++)
{
var expected = Convert.ToInt32(df1.Values[i]);
var actual = Convert.ToInt32(result.Values[i]);
Assert.Equal <int>(expected, actual);
}
}
public void TestBinaryOperatorsWithConversions()
{
var df = MakeDataFrameWithNumericColumns(10);
DataFrame tempDf = df + 1;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + (double)1);
tempDf = df + 1.1;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + 1.1);
tempDf = df + 1.1m;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + 1.1m);
Assert.True(typeof(decimal) == tempDf["Int"].DataType);
Assert.Throws <NotSupportedException>(() => df + true);
Assert.Throws <NotSupportedException>(() => df & 5);
tempDf = df - 1.1;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] - 1.1);
tempDf = df - 1.1m;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] - 1.1m);
Assert.True(typeof(decimal) == tempDf["Int"].DataType);
tempDf = df * 1.1;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] * 1.1);
tempDf = df * 1.1m;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] * 1.1m);
Assert.True(typeof(decimal) == tempDf["Int"].DataType);
tempDf = df / 1.1;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] / 1.1);
tempDf = df / 1.1m;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] / 1.1m);
Assert.True(typeof(decimal) == tempDf["Int"].DataType);
tempDf = df % 1.1;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] % 1.1);
tempDf = df % 1.1m;
Assert.Equal(tempDf[0, 0], (byte)df[0, 0] % 1.1m);
Assert.True(typeof(decimal) == tempDf["Int"].DataType);
tempDf = df == 1.1;
Assert.Equal(false, tempDf[0, 0]);
tempDf = df == 1.1m;
Assert.Equal(false, tempDf[0, 0]);
Assert.True(typeof(bool) == tempDf["Int"].DataType);
tempDf = df != 1.1;
Assert.Equal(true, tempDf[0, 0]);
tempDf = df != 1.1m;
Assert.Equal(true, tempDf[0, 0]);
tempDf = df >= 1.1;
Assert.Equal(false, tempDf[0, 0]);
tempDf = df >= 1.1m;
Assert.Equal(false, tempDf[0, 0]);
tempDf = df <= 1.1;
Assert.Equal(true, tempDf[0, 0]);
tempDf = df <= 1.1m;
Assert.Equal(true, tempDf[0, 0]);
tempDf = df > 1.1;
Assert.Equal(false, tempDf[0, 0]);
tempDf = df > 1.1m;
Assert.Equal(false, tempDf[0, 0]);
tempDf = df < 1.1;
Assert.Equal(true, tempDf[0, 0]);
tempDf = df < 1.1m;
Assert.Equal(true, tempDf[0, 0]);
Assert.Equal((byte)0, df[0, 0]);
}