Beispiel #1
0
 /// <summary>
 /// Sinks data until receives null then closes adapter
 /// </summary>
 /// <param name="data"></param>
 public override void Write(DataFrame data)
 {
     if (data == null)
     {
         _exitEvent.Set();
     }
 }
Beispiel #2
0
 /// <summary>
 /// Write a frame to the adapter
 /// </summary>
 /// <param name="frame">The frame</param>
 public override void Write(DataFrame frame)
 {
     if (frame != null)
     {
         _listener.Write(frame.ToArray(), _ep);
     }
 }
Beispiel #3
0
        /// <summary>
        /// Creates a new <see cref="DataCell"/> from specified parameters.
        /// </summary>
        /// <param name="parent">The reference to parent <see cref="DataFrame"/> of this <see cref="DataCell"/>.</param>
        /// <param name="configurationCell">The <see cref="ConfigurationCell"/> associated with this <see cref="DataCell"/>.</param>
        /// <param name="addEmptyValues">If <c>true</c>, adds empty values for each defined configuration cell definition.</param>
        public DataCell(DataFrame parent, ConfigurationCell configurationCell, bool addEmptyValues)
            : this(parent, configurationCell)
        {
            if (addEmptyValues)
            {
                int x;

                // Define needed phasor values
                for (x = 0; x < configurationCell.PhasorDefinitions.Count; x++)
                {
                    PhasorValues.Add(new PhasorValue(this, configurationCell.PhasorDefinitions[x]));
                }

                // Define a frequency and df/dt
                FrequencyValue = new FrequencyValue(this, configurationCell.FrequencyDefinition);

                // Define any analog values
                for (x = 0; x < configurationCell.AnalogDefinitions.Count; x++)
                {
                    AnalogValues.Add(new AnalogValue(this, configurationCell.AnalogDefinitions[x]));
                }

                // Define any digital values
                for (x = 0; x < configurationCell.DigitalDefinitions.Count; x++)
                {
                    DigitalValues.Add(new DigitalValue(this, configurationCell.DigitalDefinitions[x]));
                }
            }
        }
Beispiel #4
0
        public static IEnumerable<DataFrame> ParseFrames(IEnumerable<DataFrame> frames, string selectionPath, ScriptContainer container, string classname)
        {
            BasePipelineNode input;
            ParseWithPipelineNode output;
            IEnumerable<DataFrame> ret = new DataFrame[0];
            NetGraph graph = BuildGraph(container, classname, selectionPath, out input, out output);

            try
            {
                foreach (DataFrame frame in frames)
                {
                    input.Input(frame);
                }
                input.Shutdown(null);

                output.EventFlag.WaitOne(500);

                ret = output.Frames;
            }
            finally
            {
                ((IDisposable)graph).Dispose();
            }

            return ret;
        }
Beispiel #5
0
        public void ReceiveAsync(Action<string> callback, DataFrame frame = null)
        {
            var buffer = new byte[256];
            if (frame == null)
                frame = new DataFrame();

            Socket.AsyncReceive(buffer, frame, (sizeOfReceivedData, df) =>
            {
                var dataframe = (DataFrame)df;

                if (sizeOfReceivedData > 0)
                {
                    dataframe.Append(buffer);

                    if (dataframe.IsComplete)
                    {
                        var data = dataframe.ToString();

                        callback(data);

                    }
                    else // end is not is this buffer
                    {
                        ReceiveAsync(callback, dataframe); // continue to read
                    }
                }
            });
        }
        /// <summary>
        /// Constructs a new incoming packet received by a TCP connector.
        /// Use this class to receive data from a server.
        /// This is an incoming message that will remain in the client's thread pool
        /// as a job for the thread pool workers.
        /// </summary>
        /// <param name="connector">The TCP Connector where this message belongs to.</param>
        /// <param name="data">DataFrame class.</param>
        /// <param name="previousPacket">The previous incoming message of the TCP Connector.</param>
        public IncomingTCPClientPacket(TCPConnector connector, DataFrame data, IncomingTCPClientPacket previousPacket)
        {
            fConnector = connector;
            fData = data;
            fPreviousPacket = previousPacket;

            ThreadPool.QueueUserWorkItem(new WaitCallback(ReadData));
        }
Beispiel #7
0
        /// <summary>
        /// Called when a new frame arrives (just forwards)
        /// </summary>
        /// <param name="frame"></param>
        protected override void OnInput(DataFrame frame)
        {
            if (PacketDelayMs > 0)
            {
                Thread.Sleep(PacketDelayMs);
            }

            WriteOutput(frame);
        }
 /// <summary>
 /// Constructor
 /// </summary>
 /// <param name="frame">The data frame to edit</param>
 /// <param name="selectPath">Path to select a node to edit</param>
 /// <param name="sender">The sending node</param>
 /// <param name="color">The colour to show in an edit window</param>
 /// <param name="tag">The textual tag to show in an edit window</param>
 public EditPacketEventArgs(DataFrame frame, string selectPath, BasePipelineNode sender, ColorValue color, string tag)
     : base()
 {
     Frame = frame;
     SelectPath = selectPath;
     Sender = sender;
     Color = color;
     Tag = tag;
 }
 /// <summary>
 /// Constructor
 /// </summary>
 /// <param name="tag">The log tag</param>
 /// <param name="netId">The log network ID</param>
 /// <param name="frame">The log frame</param>
 /// <param name="color">The log colour</param>
 /// <param name="networkDescription">The log description</param>
 public LogPacketEventArgs(string tag, Guid netId, DataFrame frame, ColorValue color, string networkDescription)
 {
     Tag = tag;
     NetId = netId;
     Frame = frame;
     Color = color;
     NetworkDescription = networkDescription;
     Timestamp = DateTime.Now;
 }
        public void can_get_index_of_non_existant_column()
        {
            var dataFrame = new DataFrame(
                new IntColumn("Column1"),
                new StringColumn("Column2")
            );

            Assert.Equal(-1, dataFrame.GetColumnIndex("non-existant-column"));
        }
Beispiel #11
0
 public MDNS_Packet(DataFrame DataFrame)
 {
     byte[] Payload = DataFrame.Payload;
     this.TransactionId = BitConverter.ToUInt16(Payload, 0);
     this.Response = (MDNS_Response)BitConverter.ToUInt16(Payload, 2);
     this.Questions = BitConverter.ToUInt16(Payload, 4);
     this.AnswerRRs = BitConverter.ToUInt16(Payload, 6);
     this.AuthorityRRs = BitConverter.ToUInt16(Payload, 8);
     this.AdditionalRRs = BitConverter.ToUInt16(Payload, 10);
 }
Beispiel #12
0
        /// <summary>
        /// Convert a packet to a hex string format
        /// </summary>
        /// <param name="p">The packet to convert</param>
        /// <returns>The converted string</returns>
        public static string ConvertBinaryPacketToString(DataFrame p)
        {
            using (TextWriter writer = new StringWriter())
            {
                writer.WriteLine(GeneralUtils.BuildHexDump(16, p.ToArray()));
                writer.WriteLine();

                return writer.ToString();
            }
        }
Beispiel #13
0
 /// <summary>
 /// Private Constructor
 /// </summary>
 /// <param name="tag"></param>
 /// <param name="netid"></param>
 /// <param name="uuid"></param>
 /// <param name="network"></param>
 /// <param name="frame"></param>
 /// <param name="color"></param>
 /// <param name="timestamp"></param>
 public LogPacket(string tag, Guid netid, Guid uuid, string network, DataFrame frame, ColorValue color, DateTime timestamp)
 {
     Tag = tag;
     NetId = netid;
     Uuid = uuid;
     Network = network;
     Frame = frame;
     Color = color;
     Timestamp = timestamp;
 }
        public void can_get_columns()
        {
            var dataFrame = new DataFrame(
                new IntColumn("Column1"),
                new StringColumn("Column2")
            );

            var columnNames = new string[] { "Column1", "Column2" };
            Assert.Equal(columnNames, dataFrame.GetColumnNames());
        }
        public void can_get_column_index()
        {
            var dataFrame = new DataFrame(
                new IntColumn("Column1"),
                new StringColumn("Column2")
            );

            Assert.Equal(0, dataFrame.GetColumnIndex("Column1"));
            Assert.Equal(1, dataFrame.GetColumnIndex("Column2"));
        }
Beispiel #16
0
    DataFrame XorFrame(DataFrame frame, byte xorValue)
    {
        byte[] data = frame.ToArray();

        for(int i = 0; i < data.Length; ++i)
        {
            data[i] = (byte)(data[i] ^ xorValue);
        }

        return new DataFrame(data);
    }
Beispiel #17
0
	public static int Main ()
	{
		DataFrame df1 = new DataFrame ();
		DataFrame df2 = new DataFrame ();

		if (df1 != null) 
		{
			return 1;
		}

		return 0;
	}
        /// <summary>
        /// Override function called when a packet is input
        /// </summary>
        /// <param name="frame">The input frame</param>
        public override void Input(DataFrame frame)
        {
            EnsureThreadRunning();

            try
            {
                _input.Enqueue(frame);
            }
            catch (InvalidOperationException)
            { }
            catch (OperationCanceledException)
            { }
        }
Beispiel #19
0
        static void Main(string[] args)
        {
            //
            // Create a simple data frame.
            //
            var maxRange = 14;
            var dataFrame = new DataFrame(
                new IntColumn("index", Enumerable.Range(0, maxRange)),
                new DoubleColumn("Sin", Enumerable.Range(0, maxRange).Select(i => Math.Sin(i))),
                new DoubleColumn("Cos", Enumerable.Range(0, maxRange).Select(i => Math.Cos(i)))
            );

            Console.WriteLine(dataFrame.ToString());
        }
        private void Receive(UserContext context, DataFrame frame)
        {
            var bytes = frame.Payload;
            Assert.AreEqual(MessageSize, bytes.Length);

            for (var i = 0; i < bytes.Length; i++)
            {
                var value = bytes[i];
                var expectedValue = (byte)(i % 256);
                Assert.AreEqual(value, expectedValue);
            }

            countdownEvent.Signal();
        }
Beispiel #21
0
        public string Receive(DataFrame frame = null)
        {
            if (frame == null)
                frame = new DataFrame();

            var buffer = new byte[256];

            int sizeOfReceivedData =  Socket.Receive(buffer);
            frame.Append(buffer);
            if (frame.IsComplete)
                return frame.ToString();
            else
                return Receive(frame);
        }
Beispiel #22
0
 private static SimpleHyperCube convert(DataFrame dataFrame)
 {
     dynamic df = dataFrame;
      object[] colnames = Enumerable.ToArray(SymbolicExpressionExtension.AsVector(df.names));
      object[] tmp = Enumerable.ToArray(SymbolicExpressionExtension.AsVector(df.name));
      var varnames = Array.ConvertAll(tmp, x => (string)x);
      SimpleHyperCube s = new SimpleHyperCube(varnames);
      var rows = dataFrame.GetRows();
      foreach (var vn in varnames)
      {
     dynamic row = rows.First(x => ((string)((dynamic)x).name == vn));
     s.SetMinMaxValue(vn, row.min, row.max, row.value);
      }
      return s;
 }
        /// <summary>
        /// Constructs a new incoming packet for an existing TCP client connection.
        /// 
        /// Use this class to receive data from a client.
        /// This is an incoming message that will remain in the servers thread pool
        /// as a job for the thread pool workers.
        /// </summary>
        /// <param name="client">The client where this message belongs to.</param>
        /// <param name="data">DataFrame class.</param>
        /// <param name="previousPacket">The previous incoming message of the client.</param>
        public IncomingTCPClientConnectionPacket(TCPClientConnection client, DataFrame data, IncomingTCPClientConnectionPacket previousPacket)
        {
            try
            {
                fClient = client;
                fData = data;
                fPreviousPacket = previousPacket;
            }
            catch (Exception ex)
            {
                return;
            }

            ThreadPool.QueueUserWorkItem(new WaitCallback(ReadData));
        }
        /// <summary>
        /// Method called when a new frame arraives
        /// </summary>
        /// <param name="frame">The frame</param>
        protected override void OnInput(DataFrame frame)
        {
            DataNode[] nodes = frame.SelectNodes(SelectionPath);

            foreach (DataNode node in nodes)
            {
                try
                {
                    MemoryStream stm = new MemoryStream(node.ToArray());
                    DataReader reader = new DataReader(stm);
                    string name = node.Name;
                    DataKey parentKey = node.Parent;
                    node.RemoveNode();

                    while (stm.Position < stm.Length)
                    {
                        DynamicStreamDataKey2 key = new DynamicStreamDataKey2(name, Container, Graph.Logger, State);

                        reader.ByteCount = 0;
                        key.FromReader(reader);

                        // The reader clearly didn't care
                        if (reader.ByteCount == 0)
                        {
                            break;
                        }

                        parentKey.AddSubNode(key);
                        frame.Current = key;
                    }
                }
                catch (EndOfStreamException)
                {
                }
                catch (ThreadAbortException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    LogException(ex);
                }
            }

            WriteOutput(frame);
        }
Beispiel #25
0
        /// <summary>
        /// Extended match syntax, can use details from a graph and node
        /// </summary>
        /// <param name="frame">The data frame to select off</param>
        /// <param name="globalMeta">The global meta</param>
        /// <param name="meta">Meta</param>
        /// <param name="node">The current node</param>
        /// <param name="properties">Property bag</param>
        /// <param name="uuid">The uuid for private names</param>
        /// <returns>True if the filter matches</returns>
        public bool IsMatch(DataFrame frame, MetaDictionary meta, MetaDictionary globalMeta, PropertyBag properties, 
            Guid uuid, BasePipelineNode node)
        {
            bool match = false;
            DataNode[] nodes = GeneralUtils.SelectNodes(Path, frame, meta, globalMeta, properties, uuid, node);

            match = OnMatch(nodes);

            if (Invert)
            {
                return !match;
            }
            else
            {
                return match;
            }
        }
Beispiel #26
0
        /// <summary>
        /// 
        /// </summary>
        /// <param name="frame"></param>
        protected override void OnInput(DataFrame frame)
        {
            bool match = false;

            if (Filters != null)
            {
                match = Filters.IsMatch(frame, Graph.Meta, Graph.GlobalMeta, Graph.ConnectionProperties, Graph.Uuid, this);
            }

            if (match)
            {
                WriteOutput(frame, _pathName);
            }
            else
            {
                WriteOutputExclude(frame, _pathName);
            }
        }
Beispiel #27
0
        /// <summary>
        /// Set the frame to edit
        /// </summary>
        /// <param name="frame">The data frame</param>        
        /// <param name="selector">Root selection path</param>
        /// <param name="color">The colour to display the frame in (if applicable)</param>
        public void SetFrame(DataFrame frame, string selector, Color color)
        {
            DataNode node = null;
            DataNode curr = frame.Current;

            _frame = frame;
            if (!String.IsNullOrWhiteSpace(selector))
            {
                node = _frame.SelectSingleNode(selector);
            }

            if (node == null)
            {
                node = _frame.Root;
            }

            dataNodeEditorControl.SetNode(node, curr, color, ReadOnly);
        }
        public void Test_RegressionWith_SimpleKnnModel()
        {
            // Given
            var randomizer = new Random(55);
            var baseDataFrame = TestDataBuilder.BuildRandomAbstractNumericDataFrameWithRedundantAttrs(randomizer: randomizer);

            var queryDataFrame = new DataFrame(new DataTable("some data")
            {
                Columns =
                {
                    new DataColumn("F1", typeof(double)),
                    new DataColumn("F2", typeof(double)),
                    new DataColumn("F3", typeof(double)),
                    new DataColumn("F4", typeof(double)),
                    new DataColumn("F5", typeof(double))
                },
                Rows =
                {
                   new object[] { 10, 1, 1, 4, 5 },
                   new object[] { 4, 2, 1, 9, 10},
                   new object[] { 2, 1, 1, 3, 7},
                }
            });
            var expectedValues = Enumerable.Range(0, queryDataFrame.RowCount)
                .Select(
                    rowIdx =>
                        TestDataBuilder.CalcualteLinearlyDependentFeatureValue(queryDataFrame.GetNumericRowVector(rowIdx))).ToList();

            var modelBuilder = new SimpleKnnModelBuilder<double>();
            var modelParams = new KnnAdditionalParams(4, true);
            var weightingFunction = new GaussianFunction(0.3);
            var predictor = new SimpleKnnRegressor(new EuclideanDistanceMeasure(), new MinMaxNormalizer(), weightingFunction.GetValue, normalizeNumericValues: true);
            var errorMeasure = new MeanSquareError();

            // When
            var model = modelBuilder.BuildModel(baseDataFrame, "F6", modelParams);
            var results = predictor.Predict(queryDataFrame, model, "F6");

            // Then
            var mse = errorMeasure.CalculateError(Vector<double>.Build.DenseOfEnumerable(expectedValues), Vector<double>.Build.DenseOfEnumerable(results));
            Assert.IsTrue(mse < 0.55);
        }
        public void can_get_rows()
        {
            var dataFrame = new DataFrame(
                new IntColumn("Column1", new int[] { 1, 2 }),
                new StringColumn("Column2", new string[] { "A", "B" })
            );
            var rows = dataFrame.GetRows().ToArray();
            /*todo:
            Assert.Equal(2, rows.Length);
            Assert.Equal(1, rows[0].ByColumn("Column1").As<int>());
            Assert.Equal(1, rows[0].ByColumn(0).As<int>());
            Assert.Equal("A", rows[0].ByColumn("Column2").As<string>());
            Assert.Equal("A", rows[0].ByColumn(1).As<string>());

            Assert.Equal(2, rows[1].ByColumn("Column1").As<int>());
            Assert.Equal(2, rows[1].ByColumn(0).As<int>());
            Assert.Equal("B", rows[1].ByColumn("Column2").As<string>());
            Assert.Equal("B", rows[1].ByColumn(1).As<string>());
            */
        }
Beispiel #30
0
        /// <summary>
        /// Read a frame from the UDP client
        /// </summary>
        /// <returns>A data frame, null on error</returns>
        public override DataFrame Read()
        {
            DataFrame frame = null;
            IPEndPoint ep = null;

            try
            {
                byte[] data = _client.Receive(ref ep);

                if (data != null)
                {
                    frame = new DataFrame(data);
                }
            }
            catch(SocketException)
            {
            }

            return frame;
        }
Beispiel #31
0
 internal static void VerifyColumnTypes(DataFrame df, bool testArrowStringColumn = false)
 {
     foreach (DataFrameColumn column in df.Columns)
     {
         Type dataType = column.DataType;
         if (dataType == typeof(bool))
         {
             Assert.IsType <BooleanDataFrameColumn>(column);
         }
         else if (dataType == typeof(decimal))
         {
             Assert.IsType <DecimalDataFrameColumn>(column);
         }
         else if (dataType == typeof(byte))
         {
             Assert.IsType <ByteDataFrameColumn>(column);
         }
         else if (dataType == typeof(char))
         {
             Assert.IsType <CharDataFrameColumn>(column);
         }
         else if (dataType == typeof(double))
         {
             Assert.IsType <DoubleDataFrameColumn>(column);
         }
         else if (dataType == typeof(float))
         {
             Assert.IsType <SingleDataFrameColumn>(column);
         }
         else if (dataType == typeof(int))
         {
             Assert.IsType <Int32DataFrameColumn>(column);
         }
         else if (dataType == typeof(long))
         {
             Assert.IsType <Int64DataFrameColumn>(column);
         }
         else if (dataType == typeof(sbyte))
         {
             Assert.IsType <SByteDataFrameColumn>(column);
         }
         else if (dataType == typeof(short))
         {
             Assert.IsType <Int16DataFrameColumn>(column);
         }
         else if (dataType == typeof(uint))
         {
             Assert.IsType <UInt32DataFrameColumn>(column);
         }
         else if (dataType == typeof(ulong))
         {
             Assert.IsType <UInt64DataFrameColumn>(column);
         }
         else if (dataType == typeof(ushort))
         {
             Assert.IsType <UInt16DataFrameColumn>(column);
         }
         else if (dataType == typeof(string))
         {
             if (!testArrowStringColumn)
             {
                 Assert.IsType <StringDataFrameColumn>(column);
             }
             else
             {
                 Assert.IsType <ArrowStringDataFrameColumn>(column);
             }
         }
         else if (dataType == typeof(DateTime))
         {
             Assert.IsType <PrimitiveDataFrameColumn <DateTime> >(column);
         }
         else
         {
             throw new NotImplementedException("Unit test has to be updated");
         }
     }
 }
Beispiel #32
0
        public void TestComputations()
        {
            DataFrame df = MakeDataFrameWithAllColumnTypes(10);

            df["Int"][0] = -10;
            Assert.Equal(-10, df["Int"][0]);

            df["Int"].Abs();
            Assert.Equal(10, df["Int"][0]);

            Assert.Throws <NotSupportedException>(() => df["Byte"].All());
            Assert.Throws <NotSupportedException>(() => df["Byte"].Any());
            Assert.Throws <NotSupportedException>(() => df["Char"].All());
            Assert.Throws <NotSupportedException>(() => df["Char"].Any());
            Assert.Throws <NotSupportedException>(() => df["Decimal"].All());
            Assert.Throws <NotSupportedException>(() => df["Decimal"].Any());
            Assert.Throws <NotSupportedException>(() => df["Double"].All());
            Assert.Throws <NotSupportedException>(() => df["Double"].Any());
            Assert.Throws <NotSupportedException>(() => df["Float"].All());
            Assert.Throws <NotSupportedException>(() => df["Float"].Any());
            Assert.Throws <NotSupportedException>(() => df["Int"].All());
            Assert.Throws <NotSupportedException>(() => df["Int"].Any());
            Assert.Throws <NotSupportedException>(() => df["Long"].All());
            Assert.Throws <NotSupportedException>(() => df["Long"].Any());
            Assert.Throws <NotSupportedException>(() => df["Sbyte"].All());
            Assert.Throws <NotSupportedException>(() => df["Sbyte"].Any());
            Assert.Throws <NotSupportedException>(() => df["Short"].All());
            Assert.Throws <NotSupportedException>(() => df["Short"].Any());
            Assert.Throws <NotSupportedException>(() => df["Uint"].All());
            Assert.Throws <NotSupportedException>(() => df["Uint"].Any());
            Assert.Throws <NotSupportedException>(() => df["Ulong"].All());
            Assert.Throws <NotSupportedException>(() => df["Ulong"].Any());
            Assert.Throws <NotSupportedException>(() => df["Ushort"].All());
            Assert.Throws <NotSupportedException>(() => df["Ushort"].Any());

            bool any = df["Bool"].Any();
            bool all = df["Bool"].All();

            Assert.True(any);
            Assert.False(all);

            // Test the computation results
            df["Double"][0] = 100.0;
            df["Double"].CumulativeMax();
            Assert.Equal(100.0, df["Double"][9]);

            df["Float"][0] = -10.0f;
            df["Float"].CumulativeMin();
            Assert.Equal(-10.0f, df["Float"][9]);

            df["Uint"].CumulativeProduct();
            Assert.Equal((uint)0, df["Uint"][9]);

            df["Ushort"].CumulativeSum();
            Assert.Equal((ushort)40, df["Ushort"][9]);

            Assert.Equal(100.0, df["Double"].Max());
            Assert.Equal(-10.0f, df["Float"].Min());
            Assert.Equal((uint)0, df["Uint"].Product());
            Assert.Equal((ushort)140, df["Ushort"].Sum());

            df["Double"][0] = 100.1;
            Assert.Equal(100.1, df["Double"][0]);
            df["Double"].Round();
            Assert.Equal(100.0, df["Double"][0]);

            // Test that none of the numeric column types throw
            for (int i = 0; i < df.ColumnCount; i++)
            {
                BaseColumn column = df.Column(i);
                if (column.DataType == typeof(bool))
                {
                    Assert.Throws <NotSupportedException>(() => column.CumulativeMax());
                    Assert.Throws <NotSupportedException>(() => column.CumulativeMin());
                    Assert.Throws <NotSupportedException>(() => column.CumulativeProduct());
                    Assert.Throws <NotSupportedException>(() => column.CumulativeSum());
                    Assert.Throws <NotSupportedException>(() => column.Max());
                    Assert.Throws <NotSupportedException>(() => column.Min());
                    Assert.Throws <NotSupportedException>(() => column.Product());
                    Assert.Throws <NotSupportedException>(() => column.Sum());
                    continue;
                }
                else if (column.DataType == typeof(string))
                {
                    Assert.Throws <NotImplementedException>(() => column.CumulativeMax());
                    Assert.Throws <NotImplementedException>(() => column.CumulativeMin());
                    Assert.Throws <NotImplementedException>(() => column.CumulativeProduct());
                    Assert.Throws <NotImplementedException>(() => column.CumulativeSum());
                    Assert.Throws <NotImplementedException>(() => column.Max());
                    Assert.Throws <NotImplementedException>(() => column.Min());
                    Assert.Throws <NotImplementedException>(() => column.Product());
                    Assert.Throws <NotImplementedException>(() => column.Sum());
                    continue;
                }
                column.CumulativeMax();
                column.CumulativeMin();
                column.CumulativeProduct();
                column.CumulativeSum();
                column.Max();
                column.Min();
                column.Product();
                column.Sum();
            }
        }
        public void TestDataFrameNaFunctionSignatures()
        {
            DataFrameNaFunctions dfNaFuncs = _df.Na();

            var emptyColumn = new string[] { };
            var validColumn = new string[] { "age" };

            DataFrame df = dfNaFuncs.Drop("any");

            df = dfNaFuncs.Drop("all");
            df = dfNaFuncs.Drop(emptyColumn);
            df = dfNaFuncs.Drop(validColumn);
            df = dfNaFuncs.Drop("any", emptyColumn);
            df = dfNaFuncs.Drop("all", validColumn);
            df = dfNaFuncs.Drop(20);
            df = dfNaFuncs.Drop(20, emptyColumn);
            df = dfNaFuncs.Drop(20, validColumn);

            df = dfNaFuncs.Fill(100L);
            df = dfNaFuncs.Fill(100.0);
            df = dfNaFuncs.Fill("hello");
            df = dfNaFuncs.Fill(false);
            df = dfNaFuncs.Fill(100L, emptyColumn);
            df = dfNaFuncs.Fill(100L, validColumn);
            df = dfNaFuncs.Fill(100.0, emptyColumn);
            df = dfNaFuncs.Fill(100.0, validColumn);
            df = dfNaFuncs.Fill("hello", emptyColumn);
            df = dfNaFuncs.Fill("hello", validColumn);
            df = dfNaFuncs.Fill(true, emptyColumn);
            df = dfNaFuncs.Fill(true, validColumn);
            df = dfNaFuncs.Fill(new Dictionary <string, int>()
            {
                { "age", 10 }
            });
            df = dfNaFuncs.Fill(new Dictionary <string, long>()
            {
                { "age", 10L }
            });
            df = dfNaFuncs.Fill(new Dictionary <string, double>()
            {
                { "age", 10.0 }
            });
            df = dfNaFuncs.Fill(new Dictionary <string, string>()
            {
                { "age", "name" }
            });
            df = dfNaFuncs.Fill(new Dictionary <string, bool>()
            {
                { "age", false }
            });

            var doubleReplacement = new Dictionary <double, double>()
            {
                { 1.0, 5.0 }
            };
            var boolReplacement = new Dictionary <bool, bool>()
            {
                { true, false }
            };
            var stringReplacement = new Dictionary <string, string>()
            {
                { "a", "b" }
            };

            df = dfNaFuncs.Replace("age", doubleReplacement);
            df = dfNaFuncs.Replace("age", boolReplacement);
            df = dfNaFuncs.Replace("age", stringReplacement);
            df = dfNaFuncs.Replace(emptyColumn, doubleReplacement);
            df = dfNaFuncs.Replace(validColumn, doubleReplacement);
            df = dfNaFuncs.Replace(emptyColumn, boolReplacement);
            df = dfNaFuncs.Replace(validColumn, boolReplacement);
            df = dfNaFuncs.Replace(emptyColumn, stringReplacement);
            df = dfNaFuncs.Replace(validColumn, stringReplacement);
        }
Beispiel #34
0
        public void TestSignatures()
        {
            using var tempDirectory = new TemporaryDirectory();
            string path = Path.Combine(tempDirectory.Path, "delta-table");

            DataFrame rangeRate = _spark.Range(15);

            rangeRate.Write().Format("delta").Save(path);

            DeltaTable table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(path));

            table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(_spark, path));

            Assert.IsType <bool>(DeltaTable.IsDeltaTable(_spark, path));
            Assert.IsType <bool>(DeltaTable.IsDeltaTable(path));

            Assert.IsType <DeltaTable>(table.As("oldTable"));
            Assert.IsType <DeltaTable>(table.Alias("oldTable"));
            Assert.IsType <DataFrame>(table.History());
            Assert.IsType <DataFrame>(table.History(200));
            Assert.IsType <DataFrame>(table.ToDF());

            DataFrame newTable = _spark.Range(10, 15).As("newTable");

            Assert.IsType <DeltaMergeBuilder>(
                table.Merge(newTable, Functions.Exp("oldTable.id == newTable.id")));
            DeltaMergeBuilder mergeBuilder = Assert.IsType <DeltaMergeBuilder>(
                table.Merge(newTable, "oldTable.id == newTable.id"));

            // Validate the MergeBuilder matched signatures.
            Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched());
            Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched("id = 5"));
            DeltaMergeMatchedActionBuilder matchedActionBuilder =
                Assert.IsType <DeltaMergeMatchedActionBuilder>(
                    mergeBuilder.WhenMatched(Functions.Expr("id = 5")));

            Assert.IsType <DeltaMergeBuilder>(
                matchedActionBuilder.Update(new Dictionary <string, Column>()));
            Assert.IsType <DeltaMergeBuilder>(
                matchedActionBuilder.UpdateExpr(new Dictionary <string, string>()));
            Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.UpdateAll());
            Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.Delete());

            // Validate the MergeBuilder not-matched signatures.
            Assert.IsType <DeltaMergeNotMatchedActionBuilder>(mergeBuilder.WhenNotMatched());
            Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
                mergeBuilder.WhenNotMatched("id = 5"));
            DeltaMergeNotMatchedActionBuilder notMatchedActionBuilder =
                Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
                    mergeBuilder.WhenNotMatched(Functions.Expr("id = 5")));

            Assert.IsType <DeltaMergeBuilder>(
                notMatchedActionBuilder.Insert(new Dictionary <string, Column>()));
            Assert.IsType <DeltaMergeBuilder>(
                notMatchedActionBuilder.InsertExpr(new Dictionary <string, string>()));
            Assert.IsType <DeltaMergeBuilder>(notMatchedActionBuilder.InsertAll());

            // Update and UpdateExpr should return void.
            table.Update(new Dictionary <string, Column>()
            {
            });
            table.Update(Functions.Expr("id % 2 == 0"), new Dictionary <string, Column>()
            {
            });
            table.UpdateExpr(new Dictionary <string, string>()
            {
            });
            table.UpdateExpr("id % 2 == 1", new Dictionary <string, string>()
            {
            });

            Assert.IsType <DataFrame>(table.Vacuum());
            Assert.IsType <DataFrame>(table.Vacuum(168));

            // Delete should return void.
            table.Delete("id > 10");
            table.Delete(Functions.Expr("id > 5"));
            table.Delete();

            // Load the table as a streaming source.
            Assert.IsType <DataFrame>(_spark
                                      .ReadStream()
                                      .Format("delta")
                                      .Option("path", path)
                                      .Load());
            Assert.IsType <DataFrame>(_spark.ReadStream().Format("delta").Load(path));

            // Create Parquet data and convert it to DeltaTables.
            string parquetIdentifier = $"parquet.`{path}`";

            rangeRate.Write().Mode(SaveMode.Overwrite).Parquet(path);
            Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(_spark, parquetIdentifier));
            rangeRate
            .Select(Functions.Col("id"), Functions.Expr($"(`id` + 1) AS `id_plus_one`"))
            .Write()
            .PartitionBy("id")
            .Mode(SaveMode.Overwrite)
            .Parquet(path);
            Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(
                                           _spark,
                                           parquetIdentifier,
                                           "id bigint"));
            // TODO: Test with StructType partition schema once StructType is supported.
        }
Beispiel #35
0
        public void TestTutorialScenario()
        {
            using var tempDirectory = new TemporaryDirectory();
            string path = Path.Combine(tempDirectory.Path, "delta-table");

            // Write data to a Delta table.
            DataFrame data = _spark.Range(0, 5);

            data.Write().Format("delta").Save(path);

            // Validate that data contains the the sequence [0 ... 4].
            ValidateRangeDataFrame(Enumerable.Range(0, 5), data);

            // Create a second iteration of the table.
            data = _spark.Range(5, 10);
            data.Write().Format("delta").Mode("overwrite").Save(path);

            // Load the data into a DeltaTable object.
            var deltaTable = DeltaTable.ForPath(path);

            // Validate that deltaTable contains the the sequence [5 ... 9].
            ValidateRangeDataFrame(Enumerable.Range(5, 5), deltaTable.ToDF());

            // Update every even value by adding 100 to it.
            deltaTable.Update(
                condition: Functions.Expr("id % 2 == 0"),
                set: new Dictionary <string, Column>()
            {
                { "id", Functions.Expr("id + 100") }
            });

            // Validate that deltaTable contains the the data:
            // +---+
            // | id|
            // +---+
            // |  5|
            // |  7|
            // |  9|
            // |106|
            // |108|
            // +---+
            ValidateRangeDataFrame(
                new List <int>()
            {
                5, 7, 9, 106, 108
            },
                deltaTable.ToDF());

            // Delete every even value.
            deltaTable.Delete(condition: Functions.Expr("id % 2 == 0"));

            // Validate that deltaTable contains:
            // +---+
            // | id|
            // +---+
            // |  5|
            // |  7|
            // |  9|
            // +---+
            ValidateRangeDataFrame(new List <int>()
            {
                5, 7, 9
            }, deltaTable.ToDF());

            // Upsert (merge) new data.
            DataFrame newData = _spark.Range(0, 20).As("newData").ToDF();

            deltaTable.As("oldData")
            .Merge(newData, "oldData.id = newData.id")
            .WhenMatched()
            .Update(
                new Dictionary <string, Column>()
            {
                { "id", Functions.Col("newData.id") }
            })
            .WhenNotMatched()
            .InsertExpr(new Dictionary <string, string>()
            {
                { "id", "newData.id" }
            })
            .Execute();

            // Validate that the resulTable contains the the sequence [0 ... 19].
            ValidateRangeDataFrame(Enumerable.Range(0, 20), deltaTable.ToDF());
        }
Beispiel #36
0
 internal DataStreamWriter(JvmObjectReference jvmObject, DataFrame df)
 {
     Reference = jvmObject;
     _df       = df;
 }
Beispiel #37
0
 private void OnSendToWebSocket(DataFrame data, HttpRequest request)
 {
     data.Send(request.Session);
 }
Beispiel #38
0
 public virtual object FrameDeserialize(DataFrame data, PipeStream stream)
 {
     return(stream.ReadString((int)data.Length));
 }
Beispiel #39
0
        public ActionResult ExecuteWS(HttpRequest request, DataFrame dataFrame)
        {
            JToken dataToken = (JToken)Newtonsoft.Json.JsonConvert.DeserializeObject((string)dataFrame.Body);

            return(this.mActionFactory.ExecuteWithWS(request, this, dataToken));
        }
        // STATUS [ July 9, 2019 ] : this works
        public string[] GetColumnNames(DataFrame df)
        {
            var columnHeaders = df.ColumnNames;

            return(columnHeaders);
        }
Beispiel #41
0
        public void ARIM_Differencing_Test()
        {
            var dict = new Dictionary <string, List <object> >
            {
                { "a", new List <object>()
                  {
                      1, 2, 3, 4, 5, 6
                  } },
                { "b", new List <object>()
                  {
                      1, 1, 2, 3, 5, 8
                  } },
                { "c", new List <object>()
                  {
                      1, 4, 9, 16, 25, 36
                  } },
            };
            //
            var df = new DataFrame(dict);

            //seasonal differencing period 1
            var newDf = df.Diff(step: 1);

            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, 1, 1, 1, 1, 1
            }, newDf["a"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, 0, 1, 1, 2, 3
            }, newDf["b"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, 3, 5, 7, 9, 11
            }, newDf["c"]);

            //seasonal differencing period 2
            newDf = df.Diff(step: 2);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, 2, 2, 2, 2
            }, newDf["a"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, 1, 2, 3, 5
            }, newDf["b"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, 8, 12, 16, 20
            }, newDf["c"]);

            //seasonal differencing period 3
            newDf = df.Diff(step: 3);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 3, 3, 3
            }, newDf["a"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 2, 4, 6
            }, newDf["b"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 15, 21, 27
            }, newDf["c"]);
        }
Beispiel #42
0
        public void ARIM_Diff_Recursive_Test()
        {
            var dict = new Dictionary <string, List <object> >
            {
                { "a", new List <object>()
                  {
                      1, 2, 3, 4, 5, 6
                  } },
                { "b", new List <object>()
                  {
                      1, 1, 2, 3, 5, 8
                  } },
                { "c", new List <object>()
                  {
                      1, 4, 9, 16, 25, 36
                  } },
            };
            //
            var df = new DataFrame(dict);

            //seasonal differencing period 1
            var newDf = df.Diff(1, type: DiffType.Recurrsive);

            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, 1, 1, 1, 1, 1
            }, newDf["a"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, 0, 1, 1, 2, 3
            }, newDf["b"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, 3, 5, 7, 9, 11
            }, newDf["c"]);

            //seasonal differencing period 2
            newDf = df.Diff(step: 2, type: DiffType.Recurrsive);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, 0, 0, 0, 0
            }, newDf["a"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, 1, 0, 1, 1
            }, newDf["b"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, 2, 2, 2, 2
            }, newDf["c"]);

            //seasonal differencing period 3
            newDf = df.Diff(step: 3, type: DiffType.Recurrsive);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 0, 0, 0
            }, newDf["a"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, -1, 1, 0
            }, newDf["b"]);
            Assert.Equal(new List <object>()
            {
                DataFrame.NAN, DataFrame.NAN, DataFrame.NAN, 0, 0, 0
            }, newDf["c"]);
        }
Beispiel #43
0
 /// <summary>
 /// Executes the <see cref="Bucketizer"/> and transforms the DataFrame to include the new
 /// column or columns with the bucketed data.
 /// </summary>
 /// <param name="source">The DataFrame to add the bucketed data to</param>
 /// <returns>
 /// <see cref="DataFrame"/> containing the original data and the new bucketed columns
 /// </returns>
 public DataFrame Transform(DataFrame source)
 {
     return(new DataFrame((JvmObjectReference)_jvmObject.Invoke("transform", source)));
 }
Beispiel #44
0
        public void SortBy_QuickSort_Test01()
        {
            var dict = new Dictionary <string, List <object> >
            {
                { "col1", new List <object>()
                  {
                      1, 31, 41, 51, 61, 11, 21, 71, 81, 91
                  } },
                { "col2", new List <object>()
                  {
                      2, 32, 42, 52, 62, 12, 22, 72, 82, 92
                  } },
                { "col3", new List <object>()
                  {
                      3, 43, 33, 63, 53, 13, 23, 73, 83, 93
                  } },
                { "col4", new List <object>()
                  {
                      4, 54, 44, 34, 64, 14, 24, 74, 84, 94
                  } },
            };

            //
            DataFrame.qsAlgo = true;
            var df = new DataFrame(dict);

            var dict1 = new Dictionary <string, List <object> >
            {
                { "col1", new List <object>()
                  {
                      1, 11, 21, 31, 41, 51, 61, 71, 81, 91
                  } },
                { "col2", new List <object>()
                  {
                      2, 12, 22, 32, 42, 52, 62, 72, 82, 92
                  } },
                { "col3", new List <object>()
                  {
                      3, 13, 23, 43, 33, 63, 53, 73, 83, 93
                  } },
                { "col4", new List <object>()
                  {
                      4, 14, 24, 54, 44, 34, 64, 74, 84, 94
                  } },
            };
            var df1 = new DataFrame(dict1);

            var result = df.SortBy(new string[] { "col1", "col2", "col3", "col4" });
            var STR    = result.ToStringBuilder();


            var resultIndex = new object[] { 0, 5, 6, 1, 2, 3, 4, 7, 8, 9 };

            Assert.Equal(result.Index, result.Index);


            for (int i = 0; i < result.Values.Count; i++)
            {
                var expected = Convert.ToInt32(df1.Values[i]);
                var actual   = Convert.ToInt32(result.Values[i]);
                Assert.Equal <int>(expected, actual);
            }
        }
Beispiel #45
0
        public void TestBinaryOperatorsWithConversions()
        {
            var df = MakeDataFrameWithNumericColumns(10);

            DataFrame tempDf = df + 1;

            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + (double)1);
            tempDf = df + 1.1;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + 1.1);
            tempDf = df + 1.1m;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] + 1.1m);
            Assert.True(typeof(decimal) == tempDf["Int"].DataType);

            Assert.Throws <NotSupportedException>(() => df + true);
            Assert.Throws <NotSupportedException>(() => df & 5);

            tempDf = df - 1.1;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] - 1.1);
            tempDf = df - 1.1m;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] - 1.1m);
            Assert.True(typeof(decimal) == tempDf["Int"].DataType);

            tempDf = df * 1.1;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] * 1.1);
            tempDf = df * 1.1m;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] * 1.1m);
            Assert.True(typeof(decimal) == tempDf["Int"].DataType);

            tempDf = df / 1.1;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] / 1.1);
            tempDf = df / 1.1m;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] / 1.1m);
            Assert.True(typeof(decimal) == tempDf["Int"].DataType);

            tempDf = df % 1.1;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] % 1.1);
            tempDf = df % 1.1m;
            Assert.Equal(tempDf[0, 0], (byte)df[0, 0] % 1.1m);
            Assert.True(typeof(decimal) == tempDf["Int"].DataType);

            tempDf = df == 1.1;
            Assert.Equal(false, tempDf[0, 0]);
            tempDf = df == 1.1m;
            Assert.Equal(false, tempDf[0, 0]);
            Assert.True(typeof(bool) == tempDf["Int"].DataType);

            tempDf = df != 1.1;
            Assert.Equal(true, tempDf[0, 0]);
            tempDf = df != 1.1m;
            Assert.Equal(true, tempDf[0, 0]);

            tempDf = df >= 1.1;
            Assert.Equal(false, tempDf[0, 0]);
            tempDf = df >= 1.1m;
            Assert.Equal(false, tempDf[0, 0]);

            tempDf = df <= 1.1;
            Assert.Equal(true, tempDf[0, 0]);
            tempDf = df <= 1.1m;
            Assert.Equal(true, tempDf[0, 0]);

            tempDf = df > 1.1;
            Assert.Equal(false, tempDf[0, 0]);
            tempDf = df > 1.1m;
            Assert.Equal(false, tempDf[0, 0]);

            tempDf = df < 1.1;
            Assert.Equal(true, tempDf[0, 0]);
            tempDf = df < 1.1m;
            Assert.Equal(true, tempDf[0, 0]);

            Assert.Equal((byte)0, df[0, 0]);
        }