/// <summary> /// Sinks data until receives null then closes adapter /// </summary> /// <param name="data"></param> public override void Write(DataFrame data) { if (data == null) { _exitEvent.Set(); } }
/// <summary> /// Write a frame to the adapter /// </summary> /// <param name="frame">The frame</param> public override void Write(DataFrame frame) { if (frame != null) { _listener.Write(frame.ToArray(), _ep); } }
/// <summary> /// Creates a new <see cref="DataCell"/> from specified parameters. /// </summary> /// <param name="parent">The reference to parent <see cref="DataFrame"/> of this <see cref="DataCell"/>.</param> /// <param name="configurationCell">The <see cref="ConfigurationCell"/> associated with this <see cref="DataCell"/>.</param> /// <param name="addEmptyValues">If <c>true</c>, adds empty values for each defined configuration cell definition.</param> public DataCell(DataFrame parent, ConfigurationCell configurationCell, bool addEmptyValues) : this(parent, configurationCell) { if (addEmptyValues) { int x; // Define needed phasor values for (x = 0; x < configurationCell.PhasorDefinitions.Count; x++) { PhasorValues.Add(new PhasorValue(this, configurationCell.PhasorDefinitions[x])); } // Define a frequency and df/dt FrequencyValue = new FrequencyValue(this, configurationCell.FrequencyDefinition); // Define any analog values for (x = 0; x < configurationCell.AnalogDefinitions.Count; x++) { AnalogValues.Add(new AnalogValue(this, configurationCell.AnalogDefinitions[x])); } // Define any digital values for (x = 0; x < configurationCell.DigitalDefinitions.Count; x++) { DigitalValues.Add(new DigitalValue(this, configurationCell.DigitalDefinitions[x])); } } }
public static IEnumerable<DataFrame> ParseFrames(IEnumerable<DataFrame> frames, string selectionPath, ScriptContainer container, string classname) { BasePipelineNode input; ParseWithPipelineNode output; IEnumerable<DataFrame> ret = new DataFrame[0]; NetGraph graph = BuildGraph(container, classname, selectionPath, out input, out output); try { foreach (DataFrame frame in frames) { input.Input(frame); } input.Shutdown(null); output.EventFlag.WaitOne(500); ret = output.Frames; } finally { ((IDisposable)graph).Dispose(); } return ret; }
public void ReceiveAsync(Action<string> callback, DataFrame frame = null) { var buffer = new byte[256]; if (frame == null) frame = new DataFrame(); Socket.AsyncReceive(buffer, frame, (sizeOfReceivedData, df) => { var dataframe = (DataFrame)df; if (sizeOfReceivedData > 0) { dataframe.Append(buffer); if (dataframe.IsComplete) { var data = dataframe.ToString(); callback(data); } else // end is not is this buffer { ReceiveAsync(callback, dataframe); // continue to read } } }); }
/// <summary> /// Constructs a new incoming packet received by a TCP connector. /// Use this class to receive data from a server. /// This is an incoming message that will remain in the client's thread pool /// as a job for the thread pool workers. /// </summary> /// <param name="connector">The TCP Connector where this message belongs to.</param> /// <param name="data">DataFrame class.</param> /// <param name="previousPacket">The previous incoming message of the TCP Connector.</param> public IncomingTCPClientPacket(TCPConnector connector, DataFrame data, IncomingTCPClientPacket previousPacket) { fConnector = connector; fData = data; fPreviousPacket = previousPacket; ThreadPool.QueueUserWorkItem(new WaitCallback(ReadData)); }
/// <summary> /// Called when a new frame arrives (just forwards) /// </summary> /// <param name="frame"></param> protected override void OnInput(DataFrame frame) { if (PacketDelayMs > 0) { Thread.Sleep(PacketDelayMs); } WriteOutput(frame); }
/// <summary> /// Constructor /// </summary> /// <param name="frame">The data frame to edit</param> /// <param name="selectPath">Path to select a node to edit</param> /// <param name="sender">The sending node</param> /// <param name="color">The colour to show in an edit window</param> /// <param name="tag">The textual tag to show in an edit window</param> public EditPacketEventArgs(DataFrame frame, string selectPath, BasePipelineNode sender, ColorValue color, string tag) : base() { Frame = frame; SelectPath = selectPath; Sender = sender; Color = color; Tag = tag; }
/// <summary> /// Constructor /// </summary> /// <param name="tag">The log tag</param> /// <param name="netId">The log network ID</param> /// <param name="frame">The log frame</param> /// <param name="color">The log colour</param> /// <param name="networkDescription">The log description</param> public LogPacketEventArgs(string tag, Guid netId, DataFrame frame, ColorValue color, string networkDescription) { Tag = tag; NetId = netId; Frame = frame; Color = color; NetworkDescription = networkDescription; Timestamp = DateTime.Now; }
public void can_get_index_of_non_existant_column() { var dataFrame = new DataFrame( new IntColumn("Column1"), new StringColumn("Column2") ); Assert.Equal(-1, dataFrame.GetColumnIndex("non-existant-column")); }
public MDNS_Packet(DataFrame DataFrame) { byte[] Payload = DataFrame.Payload; this.TransactionId = BitConverter.ToUInt16(Payload, 0); this.Response = (MDNS_Response)BitConverter.ToUInt16(Payload, 2); this.Questions = BitConverter.ToUInt16(Payload, 4); this.AnswerRRs = BitConverter.ToUInt16(Payload, 6); this.AuthorityRRs = BitConverter.ToUInt16(Payload, 8); this.AdditionalRRs = BitConverter.ToUInt16(Payload, 10); }
/// <summary> /// Convert a packet to a hex string format /// </summary> /// <param name="p">The packet to convert</param> /// <returns>The converted string</returns> public static string ConvertBinaryPacketToString(DataFrame p) { using (TextWriter writer = new StringWriter()) { writer.WriteLine(GeneralUtils.BuildHexDump(16, p.ToArray())); writer.WriteLine(); return writer.ToString(); } }
/// <summary> /// Private Constructor /// </summary> /// <param name="tag"></param> /// <param name="netid"></param> /// <param name="uuid"></param> /// <param name="network"></param> /// <param name="frame"></param> /// <param name="color"></param> /// <param name="timestamp"></param> public LogPacket(string tag, Guid netid, Guid uuid, string network, DataFrame frame, ColorValue color, DateTime timestamp) { Tag = tag; NetId = netid; Uuid = uuid; Network = network; Frame = frame; Color = color; Timestamp = timestamp; }
public void can_get_columns() { var dataFrame = new DataFrame( new IntColumn("Column1"), new StringColumn("Column2") ); var columnNames = new string[] { "Column1", "Column2" }; Assert.Equal(columnNames, dataFrame.GetColumnNames()); }
public void can_get_column_index() { var dataFrame = new DataFrame( new IntColumn("Column1"), new StringColumn("Column2") ); Assert.Equal(0, dataFrame.GetColumnIndex("Column1")); Assert.Equal(1, dataFrame.GetColumnIndex("Column2")); }
DataFrame XorFrame(DataFrame frame, byte xorValue) { byte[] data = frame.ToArray(); for(int i = 0; i < data.Length; ++i) { data[i] = (byte)(data[i] ^ xorValue); } return new DataFrame(data); }
public static int Main () { DataFrame df1 = new DataFrame (); DataFrame df2 = new DataFrame (); if (df1 != null) { return 1; } return 0; }
/// <summary> /// Override function called when a packet is input /// </summary> /// <param name="frame">The input frame</param> public override void Input(DataFrame frame) { EnsureThreadRunning(); try { _input.Enqueue(frame); } catch (InvalidOperationException) { } catch (OperationCanceledException) { } }
static void Main(string[] args) { // // Create a simple data frame. // var maxRange = 14; var dataFrame = new DataFrame( new IntColumn("index", Enumerable.Range(0, maxRange)), new DoubleColumn("Sin", Enumerable.Range(0, maxRange).Select(i => Math.Sin(i))), new DoubleColumn("Cos", Enumerable.Range(0, maxRange).Select(i => Math.Cos(i))) ); Console.WriteLine(dataFrame.ToString()); }
private void Receive(UserContext context, DataFrame frame) { var bytes = frame.Payload; Assert.AreEqual(MessageSize, bytes.Length); for (var i = 0; i < bytes.Length; i++) { var value = bytes[i]; var expectedValue = (byte)(i % 256); Assert.AreEqual(value, expectedValue); } countdownEvent.Signal(); }
public string Receive(DataFrame frame = null) { if (frame == null) frame = new DataFrame(); var buffer = new byte[256]; int sizeOfReceivedData = Socket.Receive(buffer); frame.Append(buffer); if (frame.IsComplete) return frame.ToString(); else return Receive(frame); }
private static SimpleHyperCube convert(DataFrame dataFrame) { dynamic df = dataFrame; object[] colnames = Enumerable.ToArray(SymbolicExpressionExtension.AsVector(df.names)); object[] tmp = Enumerable.ToArray(SymbolicExpressionExtension.AsVector(df.name)); var varnames = Array.ConvertAll(tmp, x => (string)x); SimpleHyperCube s = new SimpleHyperCube(varnames); var rows = dataFrame.GetRows(); foreach (var vn in varnames) { dynamic row = rows.First(x => ((string)((dynamic)x).name == vn)); s.SetMinMaxValue(vn, row.min, row.max, row.value); } return s; }
/// <summary> /// Constructs a new incoming packet for an existing TCP client connection. /// /// Use this class to receive data from a client. /// This is an incoming message that will remain in the servers thread pool /// as a job for the thread pool workers. /// </summary> /// <param name="client">The client where this message belongs to.</param> /// <param name="data">DataFrame class.</param> /// <param name="previousPacket">The previous incoming message of the client.</param> public IncomingTCPClientConnectionPacket(TCPClientConnection client, DataFrame data, IncomingTCPClientConnectionPacket previousPacket) { try { fClient = client; fData = data; fPreviousPacket = previousPacket; } catch (Exception ex) { return; } ThreadPool.QueueUserWorkItem(new WaitCallback(ReadData)); }
/// <summary> /// Method called when a new frame arraives /// </summary> /// <param name="frame">The frame</param> protected override void OnInput(DataFrame frame) { DataNode[] nodes = frame.SelectNodes(SelectionPath); foreach (DataNode node in nodes) { try { MemoryStream stm = new MemoryStream(node.ToArray()); DataReader reader = new DataReader(stm); string name = node.Name; DataKey parentKey = node.Parent; node.RemoveNode(); while (stm.Position < stm.Length) { DynamicStreamDataKey2 key = new DynamicStreamDataKey2(name, Container, Graph.Logger, State); reader.ByteCount = 0; key.FromReader(reader); // The reader clearly didn't care if (reader.ByteCount == 0) { break; } parentKey.AddSubNode(key); frame.Current = key; } } catch (EndOfStreamException) { } catch (ThreadAbortException) { throw; } catch (Exception ex) { LogException(ex); } } WriteOutput(frame); }
/// <summary> /// Extended match syntax, can use details from a graph and node /// </summary> /// <param name="frame">The data frame to select off</param> /// <param name="globalMeta">The global meta</param> /// <param name="meta">Meta</param> /// <param name="node">The current node</param> /// <param name="properties">Property bag</param> /// <param name="uuid">The uuid for private names</param> /// <returns>True if the filter matches</returns> public bool IsMatch(DataFrame frame, MetaDictionary meta, MetaDictionary globalMeta, PropertyBag properties, Guid uuid, BasePipelineNode node) { bool match = false; DataNode[] nodes = GeneralUtils.SelectNodes(Path, frame, meta, globalMeta, properties, uuid, node); match = OnMatch(nodes); if (Invert) { return !match; } else { return match; } }
/// <summary> /// /// </summary> /// <param name="frame"></param> protected override void OnInput(DataFrame frame) { bool match = false; if (Filters != null) { match = Filters.IsMatch(frame, Graph.Meta, Graph.GlobalMeta, Graph.ConnectionProperties, Graph.Uuid, this); } if (match) { WriteOutput(frame, _pathName); } else { WriteOutputExclude(frame, _pathName); } }
/// <summary> /// Set the frame to edit /// </summary> /// <param name="frame">The data frame</param> /// <param name="selector">Root selection path</param> /// <param name="color">The colour to display the frame in (if applicable)</param> public void SetFrame(DataFrame frame, string selector, Color color) { DataNode node = null; DataNode curr = frame.Current; _frame = frame; if (!String.IsNullOrWhiteSpace(selector)) { node = _frame.SelectSingleNode(selector); } if (node == null) { node = _frame.Root; } dataNodeEditorControl.SetNode(node, curr, color, ReadOnly); }
public void Test_RegressionWith_SimpleKnnModel() { // Given var randomizer = new Random(55); var baseDataFrame = TestDataBuilder.BuildRandomAbstractNumericDataFrameWithRedundantAttrs(randomizer: randomizer); var queryDataFrame = new DataFrame(new DataTable("some data") { Columns = { new DataColumn("F1", typeof(double)), new DataColumn("F2", typeof(double)), new DataColumn("F3", typeof(double)), new DataColumn("F4", typeof(double)), new DataColumn("F5", typeof(double)) }, Rows = { new object[] { 10, 1, 1, 4, 5 }, new object[] { 4, 2, 1, 9, 10}, new object[] { 2, 1, 1, 3, 7}, } }); var expectedValues = Enumerable.Range(0, queryDataFrame.RowCount) .Select( rowIdx => TestDataBuilder.CalcualteLinearlyDependentFeatureValue(queryDataFrame.GetNumericRowVector(rowIdx))).ToList(); var modelBuilder = new SimpleKnnModelBuilder<double>(); var modelParams = new KnnAdditionalParams(4, true); var weightingFunction = new GaussianFunction(0.3); var predictor = new SimpleKnnRegressor(new EuclideanDistanceMeasure(), new MinMaxNormalizer(), weightingFunction.GetValue, normalizeNumericValues: true); var errorMeasure = new MeanSquareError(); // When var model = modelBuilder.BuildModel(baseDataFrame, "F6", modelParams); var results = predictor.Predict(queryDataFrame, model, "F6"); // Then var mse = errorMeasure.CalculateError(Vector<double>.Build.DenseOfEnumerable(expectedValues), Vector<double>.Build.DenseOfEnumerable(results)); Assert.IsTrue(mse < 0.55); }
public void can_get_rows() { var dataFrame = new DataFrame( new IntColumn("Column1", new int[] { 1, 2 }), new StringColumn("Column2", new string[] { "A", "B" }) ); var rows = dataFrame.GetRows().ToArray(); /*todo: Assert.Equal(2, rows.Length); Assert.Equal(1, rows[0].ByColumn("Column1").As<int>()); Assert.Equal(1, rows[0].ByColumn(0).As<int>()); Assert.Equal("A", rows[0].ByColumn("Column2").As<string>()); Assert.Equal("A", rows[0].ByColumn(1).As<string>()); Assert.Equal(2, rows[1].ByColumn("Column1").As<int>()); Assert.Equal(2, rows[1].ByColumn(0).As<int>()); Assert.Equal("B", rows[1].ByColumn("Column2").As<string>()); Assert.Equal("B", rows[1].ByColumn(1).As<string>()); */ }
/// <summary> /// Read a frame from the UDP client /// </summary> /// <returns>A data frame, null on error</returns> public override DataFrame Read() { DataFrame frame = null; IPEndPoint ep = null; try { byte[] data = _client.Receive(ref ep); if (data != null) { frame = new DataFrame(data); } } catch(SocketException) { } return frame; }
internal static void VerifyColumnTypes(DataFrame df, bool testArrowStringColumn = false) { foreach (DataFrameColumn column in df.Columns) { Type dataType = column.DataType; if (dataType == typeof(bool)) { Assert.IsType <BooleanDataFrameColumn>(column); } else if (dataType == typeof(decimal)) { Assert.IsType <DecimalDataFrameColumn>(column); } else if (dataType == typeof(byte)) { Assert.IsType <ByteDataFrameColumn>(column); } else if (dataType == typeof(char)) { Assert.IsType <CharDataFrameColumn>(column); } else if (dataType == typeof(double)) { Assert.IsType <DoubleDataFrameColumn>(column); } else if (dataType == typeof(float)) { Assert.IsType <SingleDataFrameColumn>(column); } else if (dataType == typeof(int)) { Assert.IsType <Int32DataFrameColumn>(column); } else if (dataType == typeof(long)) { Assert.IsType <Int64DataFrameColumn>(column); } else if (dataType == typeof(sbyte)) { Assert.IsType <SByteDataFrameColumn>(column); } else if (dataType == typeof(short)) { Assert.IsType <Int16DataFrameColumn>(column); } else if (dataType == typeof(uint)) { Assert.IsType <UInt32DataFrameColumn>(column); } else if (dataType == typeof(ulong)) { Assert.IsType <UInt64DataFrameColumn>(column); } else if (dataType == typeof(ushort)) { Assert.IsType <UInt16DataFrameColumn>(column); } else if (dataType == typeof(string)) { if (!testArrowStringColumn) { Assert.IsType <StringDataFrameColumn>(column); } else { Assert.IsType <ArrowStringDataFrameColumn>(column); } } else if (dataType == typeof(DateTime)) { Assert.IsType <PrimitiveDataFrameColumn <DateTime> >(column); } else { throw new NotImplementedException("Unit test has to be updated"); } } }
public void TestComputations() { DataFrame df = MakeDataFrameWithAllColumnTypes(10); df["Int"][0] = -10; Assert.Equal(-10, df["Int"][0]); df["Int"].Abs(); Assert.Equal(10, df["Int"][0]); Assert.Throws <NotSupportedException>(() => df["Byte"].All()); Assert.Throws <NotSupportedException>(() => df["Byte"].Any()); Assert.Throws <NotSupportedException>(() => df["Char"].All()); Assert.Throws <NotSupportedException>(() => df["Char"].Any()); Assert.Throws <NotSupportedException>(() => df["Decimal"].All()); Assert.Throws <NotSupportedException>(() => df["Decimal"].Any()); Assert.Throws <NotSupportedException>(() => df["Double"].All()); Assert.Throws <NotSupportedException>(() => df["Double"].Any()); Assert.Throws <NotSupportedException>(() => df["Float"].All()); Assert.Throws <NotSupportedException>(() => df["Float"].Any()); Assert.Throws <NotSupportedException>(() => df["Int"].All()); Assert.Throws <NotSupportedException>(() => df["Int"].Any()); Assert.Throws <NotSupportedException>(() => df["Long"].All()); Assert.Throws <NotSupportedException>(() => df["Long"].Any()); Assert.Throws <NotSupportedException>(() => df["Sbyte"].All()); Assert.Throws <NotSupportedException>(() => df["Sbyte"].Any()); Assert.Throws <NotSupportedException>(() => df["Short"].All()); Assert.Throws <NotSupportedException>(() => df["Short"].Any()); Assert.Throws <NotSupportedException>(() => df["Uint"].All()); Assert.Throws <NotSupportedException>(() => df["Uint"].Any()); Assert.Throws <NotSupportedException>(() => df["Ulong"].All()); Assert.Throws <NotSupportedException>(() => df["Ulong"].Any()); Assert.Throws <NotSupportedException>(() => df["Ushort"].All()); Assert.Throws <NotSupportedException>(() => df["Ushort"].Any()); bool any = df["Bool"].Any(); bool all = df["Bool"].All(); Assert.True(any); Assert.False(all); // Test the computation results df["Double"][0] = 100.0; df["Double"].CumulativeMax(); Assert.Equal(100.0, df["Double"][9]); df["Float"][0] = -10.0f; df["Float"].CumulativeMin(); Assert.Equal(-10.0f, df["Float"][9]); df["Uint"].CumulativeProduct(); Assert.Equal((uint)0, df["Uint"][9]); df["Ushort"].CumulativeSum(); Assert.Equal((ushort)40, df["Ushort"][9]); Assert.Equal(100.0, df["Double"].Max()); Assert.Equal(-10.0f, df["Float"].Min()); Assert.Equal((uint)0, df["Uint"].Product()); Assert.Equal((ushort)140, df["Ushort"].Sum()); df["Double"][0] = 100.1; Assert.Equal(100.1, df["Double"][0]); df["Double"].Round(); Assert.Equal(100.0, df["Double"][0]); // Test that none of the numeric column types throw for (int i = 0; i < df.ColumnCount; i++) { BaseColumn column = df.Column(i); if (column.DataType == typeof(bool)) { Assert.Throws <NotSupportedException>(() => column.CumulativeMax()); Assert.Throws <NotSupportedException>(() => column.CumulativeMin()); Assert.Throws <NotSupportedException>(() => column.CumulativeProduct()); Assert.Throws <NotSupportedException>(() => column.CumulativeSum()); Assert.Throws <NotSupportedException>(() => column.Max()); Assert.Throws <NotSupportedException>(() => column.Min()); Assert.Throws <NotSupportedException>(() => column.Product()); Assert.Throws <NotSupportedException>(() => column.Sum()); continue; } else if (column.DataType == typeof(string)) { Assert.Throws <NotImplementedException>(() => column.CumulativeMax()); Assert.Throws <NotImplementedException>(() => column.CumulativeMin()); Assert.Throws <NotImplementedException>(() => column.CumulativeProduct()); Assert.Throws <NotImplementedException>(() => column.CumulativeSum()); Assert.Throws <NotImplementedException>(() => column.Max()); Assert.Throws <NotImplementedException>(() => column.Min()); Assert.Throws <NotImplementedException>(() => column.Product()); Assert.Throws <NotImplementedException>(() => column.Sum()); continue; } column.CumulativeMax(); column.CumulativeMin(); column.CumulativeProduct(); column.CumulativeSum(); column.Max(); column.Min(); column.Product(); column.Sum(); } }
public void TestDataFrameNaFunctionSignatures() { DataFrameNaFunctions dfNaFuncs = _df.Na(); var emptyColumn = new string[] { }; var validColumn = new string[] { "age" }; DataFrame df = dfNaFuncs.Drop("any"); df = dfNaFuncs.Drop("all"); df = dfNaFuncs.Drop(emptyColumn); df = dfNaFuncs.Drop(validColumn); df = dfNaFuncs.Drop("any", emptyColumn); df = dfNaFuncs.Drop("all", validColumn); df = dfNaFuncs.Drop(20); df = dfNaFuncs.Drop(20, emptyColumn); df = dfNaFuncs.Drop(20, validColumn); df = dfNaFuncs.Fill(100L); df = dfNaFuncs.Fill(100.0); df = dfNaFuncs.Fill("hello"); df = dfNaFuncs.Fill(false); df = dfNaFuncs.Fill(100L, emptyColumn); df = dfNaFuncs.Fill(100L, validColumn); df = dfNaFuncs.Fill(100.0, emptyColumn); df = dfNaFuncs.Fill(100.0, validColumn); df = dfNaFuncs.Fill("hello", emptyColumn); df = dfNaFuncs.Fill("hello", validColumn); df = dfNaFuncs.Fill(true, emptyColumn); df = dfNaFuncs.Fill(true, validColumn); df = dfNaFuncs.Fill(new Dictionary <string, int>() { { "age", 10 } }); df = dfNaFuncs.Fill(new Dictionary <string, long>() { { "age", 10L } }); df = dfNaFuncs.Fill(new Dictionary <string, double>() { { "age", 10.0 } }); df = dfNaFuncs.Fill(new Dictionary <string, string>() { { "age", "name" } }); df = dfNaFuncs.Fill(new Dictionary <string, bool>() { { "age", false } }); var doubleReplacement = new Dictionary <double, double>() { { 1.0, 5.0 } }; var boolReplacement = new Dictionary <bool, bool>() { { true, false } }; var stringReplacement = new Dictionary <string, string>() { { "a", "b" } }; df = dfNaFuncs.Replace("age", doubleReplacement); df = dfNaFuncs.Replace("age", boolReplacement); df = dfNaFuncs.Replace("age", stringReplacement); df = dfNaFuncs.Replace(emptyColumn, doubleReplacement); df = dfNaFuncs.Replace(validColumn, doubleReplacement); df = dfNaFuncs.Replace(emptyColumn, boolReplacement); df = dfNaFuncs.Replace(validColumn, boolReplacement); df = dfNaFuncs.Replace(emptyColumn, stringReplacement); df = dfNaFuncs.Replace(validColumn, stringReplacement); }
public void TestSignatures() { using var tempDirectory = new TemporaryDirectory(); string path = Path.Combine(tempDirectory.Path, "delta-table"); DataFrame rangeRate = _spark.Range(15); rangeRate.Write().Format("delta").Save(path); DeltaTable table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(path)); table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(_spark, path)); Assert.IsType <bool>(DeltaTable.IsDeltaTable(_spark, path)); Assert.IsType <bool>(DeltaTable.IsDeltaTable(path)); Assert.IsType <DeltaTable>(table.As("oldTable")); Assert.IsType <DeltaTable>(table.Alias("oldTable")); Assert.IsType <DataFrame>(table.History()); Assert.IsType <DataFrame>(table.History(200)); Assert.IsType <DataFrame>(table.ToDF()); DataFrame newTable = _spark.Range(10, 15).As("newTable"); Assert.IsType <DeltaMergeBuilder>( table.Merge(newTable, Functions.Exp("oldTable.id == newTable.id"))); DeltaMergeBuilder mergeBuilder = Assert.IsType <DeltaMergeBuilder>( table.Merge(newTable, "oldTable.id == newTable.id")); // Validate the MergeBuilder matched signatures. Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched()); Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched("id = 5")); DeltaMergeMatchedActionBuilder matchedActionBuilder = Assert.IsType <DeltaMergeMatchedActionBuilder>( mergeBuilder.WhenMatched(Functions.Expr("id = 5"))); Assert.IsType <DeltaMergeBuilder>( matchedActionBuilder.Update(new Dictionary <string, Column>())); Assert.IsType <DeltaMergeBuilder>( matchedActionBuilder.UpdateExpr(new Dictionary <string, string>())); Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.UpdateAll()); Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.Delete()); // Validate the MergeBuilder not-matched signatures. Assert.IsType <DeltaMergeNotMatchedActionBuilder>(mergeBuilder.WhenNotMatched()); Assert.IsType <DeltaMergeNotMatchedActionBuilder>( mergeBuilder.WhenNotMatched("id = 5")); DeltaMergeNotMatchedActionBuilder notMatchedActionBuilder = Assert.IsType <DeltaMergeNotMatchedActionBuilder>( mergeBuilder.WhenNotMatched(Functions.Expr("id = 5"))); Assert.IsType <DeltaMergeBuilder>( notMatchedActionBuilder.Insert(new Dictionary <string, Column>())); Assert.IsType <DeltaMergeBuilder>( notMatchedActionBuilder.InsertExpr(new Dictionary <string, string>())); Assert.IsType <DeltaMergeBuilder>(notMatchedActionBuilder.InsertAll()); // Update and UpdateExpr should return void. table.Update(new Dictionary <string, Column>() { }); table.Update(Functions.Expr("id % 2 == 0"), new Dictionary <string, Column>() { }); table.UpdateExpr(new Dictionary <string, string>() { }); table.UpdateExpr("id % 2 == 1", new Dictionary <string, string>() { }); Assert.IsType <DataFrame>(table.Vacuum()); Assert.IsType <DataFrame>(table.Vacuum(168)); // Delete should return void. table.Delete("id > 10"); table.Delete(Functions.Expr("id > 5")); table.Delete(); // Load the table as a streaming source. Assert.IsType <DataFrame>(_spark .ReadStream() .Format("delta") .Option("path", path) .Load()); Assert.IsType <DataFrame>(_spark.ReadStream().Format("delta").Load(path)); // Create Parquet data and convert it to DeltaTables. string parquetIdentifier = $"parquet.`{path}`"; rangeRate.Write().Mode(SaveMode.Overwrite).Parquet(path); Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(_spark, parquetIdentifier)); rangeRate .Select(Functions.Col("id"), Functions.Expr($"(`id` + 1) AS `id_plus_one`")) .Write() .PartitionBy("id") .Mode(SaveMode.Overwrite) .Parquet(path); Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta( _spark, parquetIdentifier, "id bigint")); // TODO: Test with StructType partition schema once StructType is supported. }
public void TestTutorialScenario() { using var tempDirectory = new TemporaryDirectory(); string path = Path.Combine(tempDirectory.Path, "delta-table"); // Write data to a Delta table. DataFrame data = _spark.Range(0, 5); data.Write().Format("delta").Save(path); // Validate that data contains the the sequence [0 ... 4]. ValidateRangeDataFrame(Enumerable.Range(0, 5), data); // Create a second iteration of the table. data = _spark.Range(5, 10); data.Write().Format("delta").Mode("overwrite").Save(path); // Load the data into a DeltaTable object. var deltaTable = DeltaTable.ForPath(path); // Validate that deltaTable contains the the sequence [5 ... 9]. ValidateRangeDataFrame(Enumerable.Range(5, 5), deltaTable.ToDF()); // Update every even value by adding 100 to it. deltaTable.Update( condition: Functions.Expr("id % 2 == 0"), set: new Dictionary <string, Column>() { { "id", Functions.Expr("id + 100") } }); // Validate that deltaTable contains the the data: // +---+ // | id| // +---+ // | 5| // | 7| // | 9| // |106| // |108| // +---+ ValidateRangeDataFrame( new List <int>() { 5, 7, 9, 106, 108 }, deltaTable.ToDF()); // Delete every even value. deltaTable.Delete(condition: Functions.Expr("id % 2 == 0")); // Validate that deltaTable contains: // +---+ // | id| // +---+ // | 5| // | 7| // | 9| // +---+ ValidateRangeDataFrame(new List <int>() { 5, 7, 9 }, deltaTable.ToDF()); // Upsert (merge) new data. DataFrame newData = _spark.Range(0, 20).As("newData").ToDF(); deltaTable.As("oldData") .Merge(newData, "oldData.id = newData.id") .WhenMatched() .Update( new Dictionary <string, Column>() { { "id", Functions.Col("newData.id") } }) .WhenNotMatched() .InsertExpr(new Dictionary <string, string>() { { "id", "newData.id" } }) .Execute(); // Validate that the resulTable contains the the sequence [0 ... 19]. ValidateRangeDataFrame(Enumerable.Range(0, 20), deltaTable.ToDF()); }
internal DataStreamWriter(JvmObjectReference jvmObject, DataFrame df) { Reference = jvmObject; _df = df; }
private void OnSendToWebSocket(DataFrame data, HttpRequest request) { data.Send(request.Session); }
public virtual object FrameDeserialize(DataFrame data, PipeStream stream) { return(stream.ReadString((int)data.Length)); }
public ActionResult ExecuteWS(HttpRequest request, DataFrame dataFrame) { JToken dataToken = (JToken)Newtonsoft.Json.JsonConvert.DeserializeObject((string)dataFrame.Body); return(this.mActionFactory.ExecuteWithWS(request, this, dataToken)); }
// STATUS [ July 9, 2019 ] : this works public string[] GetColumnNames(DataFrame df) { var columnHeaders = df.ColumnNames; return(columnHeaders); }
public void ARIM_Differencing_Test() { var dict = new Dictionary <string, List <object> > { { "a", new List <object>() { 1, 2, 3, 4, 5, 6 } }, { "b", new List <object>() { 1, 1, 2, 3, 5, 8 } }, { "c", new List <object>() { 1, 4, 9, 16, 25, 36 } }, }; // var df = new DataFrame(dict); //seasonal differencing period 1 var newDf = df.Diff(step: 1); Assert.Equal(new List <object>() { DataFrame.NAN, 1, 1, 1, 1, 1 }, newDf["a"]); Assert.Equal(new List <object>() { DataFrame.NAN, 0, 1, 1, 2, 3 }, newDf["b"]); Assert.Equal(new List <object>() { DataFrame.NAN, 3, 5, 7, 9, 11 }, newDf["c"]); //seasonal differencing period 2 newDf = df.Diff(step: 2); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, 2, 2, 2, 2 }, newDf["a"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, 1, 2, 3, 5 }, newDf["b"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, 8, 12, 16, 20 }, newDf["c"]); //seasonal differencing period 3 newDf = df.Diff(step: 3); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 3, 3, 3 }, newDf["a"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 2, 4, 6 }, newDf["b"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 15, 21, 27 }, newDf["c"]); }
public void ARIM_Diff_Recursive_Test() { var dict = new Dictionary <string, List <object> > { { "a", new List <object>() { 1, 2, 3, 4, 5, 6 } }, { "b", new List <object>() { 1, 1, 2, 3, 5, 8 } }, { "c", new List <object>() { 1, 4, 9, 16, 25, 36 } }, }; // var df = new DataFrame(dict); //seasonal differencing period 1 var newDf = df.Diff(1, type: DiffType.Recurrsive); Assert.Equal(new List <object>() { DataFrame.NAN, 1, 1, 1, 1, 1 }, newDf["a"]); Assert.Equal(new List <object>() { DataFrame.NAN, 0, 1, 1, 2, 3 }, newDf["b"]); Assert.Equal(new List <object>() { DataFrame.NAN, 3, 5, 7, 9, 11 }, newDf["c"]); //seasonal differencing period 2 newDf = df.Diff(step: 2, type: DiffType.Recurrsive); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, 0, 0, 0, 0 }, newDf["a"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, 1, 0, 1, 1 }, newDf["b"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, 2, 2, 2, 2 }, newDf["c"]); //seasonal differencing period 3 newDf = df.Diff(step: 3, type: DiffType.Recurrsive); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 0, 0, 0 }, newDf["a"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, -1, 1, 0 }, newDf["b"]); Assert.Equal(new List <object>() { DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 0, 0, 0 }, newDf["c"]); }
/// <summary> /// Executes the <see cref="Bucketizer"/> and transforms the DataFrame to include the new /// column or columns with the bucketed data. /// </summary> /// <param name="source">The DataFrame to add the bucketed data to</param> /// <returns> /// <see cref="DataFrame"/> containing the original data and the new bucketed columns /// </returns> public DataFrame Transform(DataFrame source) { return(new DataFrame((JvmObjectReference)_jvmObject.Invoke("transform", source))); }
public void SortBy_QuickSort_Test01() { var dict = new Dictionary <string, List <object> > { { "col1", new List <object>() { 1, 31, 41, 51, 61, 11, 21, 71, 81, 91 } }, { "col2", new List <object>() { 2, 32, 42, 52, 62, 12, 22, 72, 82, 92 } }, { "col3", new List <object>() { 3, 43, 33, 63, 53, 13, 23, 73, 83, 93 } }, { "col4", new List <object>() { 4, 54, 44, 34, 64, 14, 24, 74, 84, 94 } }, }; // DataFrame.qsAlgo = true; var df = new DataFrame(dict); var dict1 = new Dictionary <string, List <object> > { { "col1", new List <object>() { 1, 11, 21, 31, 41, 51, 61, 71, 81, 91 } }, { "col2", new List <object>() { 2, 12, 22, 32, 42, 52, 62, 72, 82, 92 } }, { "col3", new List <object>() { 3, 13, 23, 43, 33, 63, 53, 73, 83, 93 } }, { "col4", new List <object>() { 4, 14, 24, 54, 44, 34, 64, 74, 84, 94 } }, }; var df1 = new DataFrame(dict1); var result = df.SortBy(new string[] { "col1", "col2", "col3", "col4" }); var STR = result.ToStringBuilder(); var resultIndex = new object[] { 0, 5, 6, 1, 2, 3, 4, 7, 8, 9 }; Assert.Equal(result.Index, result.Index); for (int i = 0; i < result.Values.Count; i++) { var expected = Convert.ToInt32(df1.Values[i]); var actual = Convert.ToInt32(result.Values[i]); Assert.Equal <int>(expected, actual); } }
public void TestBinaryOperatorsWithConversions() { var df = MakeDataFrameWithNumericColumns(10); DataFrame tempDf = df + 1; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + (double)1); tempDf = df + 1.1; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + 1.1); tempDf = df + 1.1m; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + 1.1m); Assert.True(typeof(decimal) == tempDf["Int"].DataType); Assert.Throws <NotSupportedException>(() => df + true); Assert.Throws <NotSupportedException>(() => df & 5); tempDf = df - 1.1; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] - 1.1); tempDf = df - 1.1m; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] - 1.1m); Assert.True(typeof(decimal) == tempDf["Int"].DataType); tempDf = df * 1.1; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] * 1.1); tempDf = df * 1.1m; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] * 1.1m); Assert.True(typeof(decimal) == tempDf["Int"].DataType); tempDf = df / 1.1; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] / 1.1); tempDf = df / 1.1m; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] / 1.1m); Assert.True(typeof(decimal) == tempDf["Int"].DataType); tempDf = df % 1.1; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] % 1.1); tempDf = df % 1.1m; Assert.Equal(tempDf[0, 0], (byte)df[0, 0] % 1.1m); Assert.True(typeof(decimal) == tempDf["Int"].DataType); tempDf = df == 1.1; Assert.Equal(false, tempDf[0, 0]); tempDf = df == 1.1m; Assert.Equal(false, tempDf[0, 0]); Assert.True(typeof(bool) == tempDf["Int"].DataType); tempDf = df != 1.1; Assert.Equal(true, tempDf[0, 0]); tempDf = df != 1.1m; Assert.Equal(true, tempDf[0, 0]); tempDf = df >= 1.1; Assert.Equal(false, tempDf[0, 0]); tempDf = df >= 1.1m; Assert.Equal(false, tempDf[0, 0]); tempDf = df <= 1.1; Assert.Equal(true, tempDf[0, 0]); tempDf = df <= 1.1m; Assert.Equal(true, tempDf[0, 0]); tempDf = df > 1.1; Assert.Equal(false, tempDf[0, 0]); tempDf = df > 1.1m; Assert.Equal(false, tempDf[0, 0]); tempDf = df < 1.1; Assert.Equal(true, tempDf[0, 0]); tempDf = df < 1.1m; Assert.Equal(true, tempDf[0, 0]); Assert.Equal((byte)0, df[0, 0]); }