Пример #1
0
        public DenseMatrix NormalizeData(DenseMatrix data)
        {
            var normalizedData = new DenseMatrix(data.RowCount, data.ColumnCount);

            for (int i = 0; i < data.RowCount; i++)
            {
                normalizedData.SetRow(i, normalizeArrayInput[i].Process(data.Row(i).ToArray()));
            }

            return normalizedData;
        }
Пример #2
0
        private static double Error(DenseVector weights, DenseMatrix xM, DenseVector yV)
        {
            Debug.Assert(xM.RowCount == yV.Count);
              Debug.Assert(xM.ColumnCount == weights.Count);

              double wrong = 0.0;
              for (int i = 0; i < xM.RowCount; i++)
              {
            var p = xM.Row(i) * weights;
            if ((p < 0 ? -1 : 1) != yV[i])
              wrong += 1;
              }
              return wrong / xM.RowCount;
        }
Пример #3
0
		public void Smooth(ref double[,] inputValues)
		{
			// TODO: Using the matrix works, but does a lot of data accesses. Can improve by working out all the data access myself? I might be able to cut down on number of data accesses, but not sure.
		    var inputMatrix = new DenseMatrix(inputValues.GetLength(0), inputValues.GetLength(1));

			for (int i = 0; i < inputMatrix.RowCount; i++)
			{
				inputMatrix.SetRow(i, Smooth(inputMatrix.Row(i).ToArray()));
			}

			for (int i = 0; i < inputMatrix.ColumnCount; i++)
			{
				inputMatrix.SetColumn(i, Smooth(inputMatrix.Column(i).ToArray()));
			}

			inputValues = inputMatrix.ToArray();
		}
Пример #4
0
        public void FinishAndProcess()
        {
            var priceData = new DenseMatrix(symbols.Length, numTicks);

            for (int j = 0; j < symbols.Length; j++)
            {
                SortedList<DateTime, Tick> d = mktData[j].data.Data;
                for (int k = 0; k < d.Count; k++)
                {
                    priceData[j, k] = d.Values[k].BidClose;
                }
            }

            for (int i = 0; i < symbols.Length; i++)
            {
                results[i] = function((DenseVector) priceData.Row(i));
            }
        }
Пример #5
0
        public static Dictionary<int, double> GetTopK2(DenseMatrix userSimilarities, int uidTarget, int K)
        {
            Dictionary<int, double> topK = new Dictionary<int, double>(K);
            Vector<double> uidSimilarities = userSimilarities.Row(uidTarget);

            double minSimilarity = double.MinValue;
            int minUid = int.MinValue;
            foreach (Tuple<int, double> entry in uidSimilarities.EnumerateIndexed(Zeros.AllowSkip))
            {
                int uid = entry.Item1;
                double similarity = entry.Item2;
                if (uid == uidTarget) { continue; } // A user is not a neighbor of himself
                if (topK.Count < K)  // Fill the top K list untill it is full
                {
                    topK[uid] = similarity;
                    if (topK.Count == K)
                    {
                        // Find the least similar neighbor when it is full
                        minUid = topK.Aggregate((l, r) => l.Value < r.Value ? l : r).Key;
                        minSimilarity = topK[minUid];
                    }
                }
                else if (similarity > minSimilarity)
                {
                    // Replace the least similar neighbor
                    topK.Remove(minUid);    // The first time it does nothing as the minUid is not set
                    topK[uid] = similarity;

                    // Find the least similar neighbor
                    minUid = topK.Aggregate((l, r) => l.Value < r.Value ? l : r).Key;
                    minSimilarity = topK[minUid];
                }
            }

            return topK;
        }
Пример #6
0
        public static void Process(FXSession session, string symbol1, string symbol2, string timeframe, int length)
        {


            HistoricPriceEngine h1 = new HistoricPriceEngine(session);
            h1.GetLongHistoricPrices(symbol1, timeframe, length);

            while (!h1.Complete)
            {
                Thread.Sleep(100);
            }

            HistoricPriceEngine h2 = new HistoricPriceEngine(session);
            h2.GetLongHistoricPrices(symbol2, timeframe, length);

            while (!h2.Complete)
            {
                Thread.Sleep(100);
            }
            //-----------------------

            var dateTimeList = new SortedList<DateTime, int>();

            Quantum q1 = h1.Data;
            Quantum q2 = h2.Data;


            var priceData = new DenseMatrix(2, q1.Data.Count);

            for (int j = 0; j < ((q1.Data.Count <= q2.Data.Count)?q1.Data.Count:q2.Data.Count); j++ )
            {
                dateTimeList.Add(q1.Data.Values[j].Time, 1);
                priceData[0, j] = q1.Data.Values[j].BidClose;
                priceData[1, j] = q2.Data.Values[j].BidClose;
            }

            Vector<double> price1 = priceData.Row(0);
            Vector<double> price2 = priceData.Row(1);
            //Statistics.ApplyFunction((DenseVector)price1, Math.Log);
            //Statistics.ApplyFunction((DenseVector)price2, Math.Log);

            DenseVector norm1 = price1.ToArray().NormalizeZScore();
            DenseVector norm2 = price2.ToArray().NormalizeZScore();

            var newsym = new string[] {symbol1, symbol2, "spread"};

            var m = new DenseMatrix(6, norm1.Count);
            m.SetRow(0, norm1);
            m.SetRow(1, norm2);
            m.SetRow(2, (norm1 - norm2).ToArray().NormalizeZScore());

            string filename = symbol1.Replace('/', '_') + "-" + symbol2.Replace('/', '_') + ".html";


            Visualize.GenerateMultiPaneGraph(newsym, dateTimeList.Keys.ToArray(), m, QSConstants.DEFAULT_DATA_FILEPATH + filename,
                new ChartOption[]{new ChartOption(), new ChartOption(){Layover = true, YPosition = 0}, new ChartOption(){YPosition = 1} }, null, filename + ".json");

            FileUpload.UploadFileToFTP(QSConstants.DEFAULT_DATA_FILEPATH + filename, filename);
            FileUpload.UploadFileToFTP(QSConstants.DEFAULT_DATA_FILEPATH + filename + ".json", filename + ".json");

            double Spread = m[2, m.ColumnCount - 1];

            if (Spread > 2.0 && m[2, m.ColumnCount - 2] <= 2.0)
                Emailer.SendEmail(symbol1 + "-" + symbol2 + " Spread Above 2.0", "Test");

            if (Spread < -2.0 && m[2, m.ColumnCount - 2] >= -2.0)
                Emailer.SendEmail(symbol1 + "-" + symbol2 + " Spread Below -2.0", "Test");

        }
Пример #7
0
        public void Train(DenseMatrix X, DenseVector d, DenseVector Kd)
        {
            int R = X.RowCount;
            int N = X.ColumnCount;
            int U = 0; //the number of neurons in the structure


            var c = new DenseMatrix(R, 1);
            var sigma = new DenseMatrix(R, 1);

            var Q = new DenseMatrix((R + 1), (R + 1));
            var O = new DenseMatrix(1, (R + 1));
            var pT_n = new DenseMatrix((R + 1), 1);

            double maxPhi = 0;
            int maxIndex;

            var Psi = new DenseMatrix(N, 1);

            Console.WriteLine("Running...");
            //for each observation n in X
            for (int i = 0; i < N; i++)
            {
                Console.WriteLine(100*(i/(double) N) + "%");

                var x = new DenseVector(R);
                X.Column(i, x);

                //if there are neurons in structure,
                //update structure recursively.
                if (U == 0)
                {
                    c = (DenseMatrix) x.ToColumnMatrix();
                    sigma = new DenseMatrix(R, 1, SigmaZero);
                    U = 1;
                    Psi = CalculatePsi(X, c, sigma);
                    UpdateStructure(X, Psi, d, ref Q, ref O);
                    pT_n =
                        (DenseMatrix)
                            (CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) Psi.Row(i).ToRowMatrix()))
                                .Transpose();
                }
                else
                {
                    StructureRecurse(X, Psi, d, i, ref Q, ref O, ref pT_n);
                }


                bool KeepSpinning = true;
                while (KeepSpinning)
                {
                    //Calculate the error and if-part criteria
                    double ee = pT_n.Multiply(O)[0, 0];

                    double approximationError = Math.Abs(d[i] - ee);

                    DenseVector Phi;
                    double SumPhi;
                    CalculatePhi(x, c, sigma, out Phi, out SumPhi);

                    maxPhi = Phi.Maximum();
                    maxIndex = Phi.MaximumIndex();

                    if (approximationError > delta)
                    {
                        if (maxPhi < threshold)
                        {
                            var tempSigma = new DenseVector(R);
                            sigma.Column(maxIndex, tempSigma);

                            double minSigma = tempSigma.Minimum();
                            int minIndex = tempSigma.MinimumIndex();
                            sigma[minIndex, maxIndex] = k_sigma*minSigma;
                            Psi = CalculatePsi(X, c, sigma);
                            UpdateStructure(X, Psi, d, ref Q, ref O);
                            var psi = new DenseVector(Psi.ColumnCount);
                            Psi.Row(i, psi);

                            pT_n =
                                (DenseMatrix)
                                    CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix())
                                        .Transpose();
                        }
                        else
                        {
                            //add a new neuron and update strucutre

                            double distance = 0;
                            var cTemp = new DenseVector(R);
                            var sigmaTemp = new DenseVector(R);

                            //foreach input variable
                            for (int j = 0; j < R; j++)
                            {
                                distance = Math.Abs(x[j] - c[j, 0]);
                                int distanceIndex = 0;

                                //foreach neuron past 1
                                for (int k = 1; k < U; k++)
                                {
                                    if ((Math.Abs(x[j] - c[j, k])) < distance)
                                    {
                                        distanceIndex = k;
                                        distance = Math.Abs(x[j] - c[j, k]);
                                    }
                                }

                                if (distance < Kd[j])
                                {
                                    cTemp[j] = c[j, distanceIndex];
                                    sigmaTemp[j] = sigma[j, distanceIndex];
                                }
                                else
                                {
                                    cTemp[j] = x[j];
                                    sigmaTemp[j] = distance;
                                }
                            }
                            //end foreach

                            c = (DenseMatrix) c.InsertColumn(c.ColumnCount - 1, cTemp);
                            sigma = (DenseMatrix) sigma.InsertColumn(sigma.ColumnCount - 1, sigmaTemp);
                            Psi = CalculatePsi(X, c, sigma);
                            UpdateStructure(X, Psi, d, ref Q, ref O);
                            U++;
                            KeepSpinning = false;
                        }
                    }
                    else
                    {
                        if (maxPhi < threshold)
                        {
                            var tempSigma = new DenseVector(R);
                            sigma.Column(maxIndex, tempSigma);

                            double minSigma = tempSigma.Minimum();
                            int minIndex = tempSigma.MinimumIndex();
                            sigma[minIndex, maxIndex] = k_sigma*minSigma;
                            Psi = CalculatePsi(X, c, sigma);
                            UpdateStructure(X, Psi, d, ref Q, ref O);
                            var psi = new DenseVector(Psi.ColumnCount);
                            Psi.Row(i, psi);

                            pT_n =
                                (DenseMatrix)
                                    CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix())
                                        .Transpose();
                        }
                        else
                        {
                            KeepSpinning = false;
                        }
                    }
                }
            }

            out_C = c;
            out_O = O;
            out_Sigma = sigma;

            Console.WriteLine("Done.");
        }
Пример #8
0
        public void StructureRecurse(DenseMatrix X, DenseMatrix Psi, DenseVector d, int n, ref DenseMatrix Q,
            ref DenseMatrix O, ref DenseMatrix pT_n)
        {
            //O = O(t-1) O_enxt = O(t)
            //o should be a column vector ( in matrix form)
            var x = new DenseVector(X.RowCount);
            var psi = new DenseVector(Psi.ColumnCount);

            X.Column(n, x);
            Psi.Row(n, psi);

            DenseMatrix p_n = CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix());

            pT_n = (DenseMatrix) p_n.Transpose();

            double ee = Math.Abs(d[n] - (pT_n.Multiply(O))[0, 0]);
            double temp = 1 + (pT_n.Multiply(Q)).Multiply(p_n)[0, 0];
            double ae = Math.Abs(ee/temp);

            if (ee >= ae)
            {
                var L = (DenseMatrix) Q.Multiply(p_n).Multiply(1/temp);
                Q = (DenseMatrix) ((DenseMatrix.Identity(Q.RowCount).Subtract(L.Multiply(pT_n))).Multiply(Q));
                O = (DenseMatrix) O.Add(L*ee);
            }
            else
            {
                Q = (DenseMatrix) DenseMatrix.Identity(Q.RowCount).Multiply(Q);
            }
        }
Пример #9
0
        public void Predict(DenseMatrix newPredictData, double[] newPredictPrices)
        {
            double error = 0;
            int c = 0;

            var newNormalizedPredictData = new DenseMatrix(newPredictData.RowCount, newPredictData.ColumnCount,
                double.NaN);

            for (int i = 0; i < newPredictData.RowCount; i++)
            {
                newNormalizedPredictData.SetRow(i, normalizeArrayInput[i].Process(newPredictData.Row(i).ToArray()));
            }

            double[] normalizedPrices = normalizeArrayOutput.Process(newPredictPrices);

            var d = new DenseMatrix(2, normalizedPrices.Length + 1, double.NaN);
            int count = 0;
            for (int i = 0; i < normalizedPrices.Length; i++)
            {
                // calculate based on actual data
                IMLData input = new BasicMLData(inputs);
                for (int j = 0; j < input.Count; j++)
                {
                    input.Data[j] = newNormalizedPredictData[j, i];
                }

                IMLData output = network.Compute(input);
                double prediction = output.Data[0];


                error +=
                    Math.Pow(
                        (normalizeArrayOutput.Stats.DeNormalize(prediction) - newPredictPrices[i])/newPredictPrices[i],
                        2);
                c++;
                d[0, count] = newPredictPrices[i];
                d[1, count] = normalizeArrayOutput.Stats.DeNormalize(prediction);
                count++;
            }

            /////////////////////////////////////////////////////////////////

            IMLData input1 = new BasicMLData(inputs);
            for (int j = 0; j < input1.Count; j++)
            {
                input1.Data[j] = newNormalizedPredictData[j, newNormalizedPredictData.ColumnCount - 1];
            }

            IMLData output1 = network.Compute(input1);
            d[1, count] = normalizeArrayOutput.Stats.DeNormalize(output1.Data[0]);


            /////////////////////////////////////////////////////////////////


            error /= c;
            error = Math.Pow(error, .5);
            Console.WriteLine(error);

            string[] symbols = {"actual", "predicted"};
            Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0),
                "C:\\Sangar\\resultfinal.html");

            outputCorre =
                StatisticsExtension.Correlation(d.Row(0).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn(),
                    d.Row(1).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn());

            Console.WriteLine("ST2 Correlation: " + outputCorre);

            outputRMSE = error;

            Console.WriteLine("Predicted return for D+1:" +
                              (d[1, d.ColumnCount - 1] - d[1, d.ColumnCount - 2])/d[1, d.ColumnCount - 2]*100 +
                              " percent");
        }
Пример #10
0
        public Krige(DenseMatrix xx, DenseMatrix yy, Powvargram vg)
        {
            // Initializer
            x = xx;
            vgram = vg;
            n = xx.RowCount;
            d = xx.ColumnCount;

            dstar = new DenseMatrix(n + 1, 1);
            vstar = new DenseMatrix(n + 1, 1);
            v = new DenseMatrix(n + 1, n + 1);
            y = new DenseMatrix(n + 1, n);

            // Building matrix V,Y
            for (int i = 0; i < n; i++)
            {
                y[i, 0] = yy[i, 0];
                for (int j = i; j < n; j++)
                {
                    double vvv = vgram.GetV((x.Row(i) - x.Row(j)).Norm(2));
                    v[i, j] = vvv;
                    v[j, i] = vvv;
                }
                v[i, n] = 1.0;
                v[n, i] = 1.0;
            }
            v[n, n] = 0.0;
            y[n, 0] = 0.0;

            vi = v.Inverse();
            yvi = vi.Transpose() * y;
        }
Пример #11
0
        static Tuple<double, double> RunPLAvsSVM(int experiments, int points)
        {
            const int TEST_POINTS = 10000;
              Random rnd = new Random();

              long svmWins = 0, svCount = 0;
              for (int i = 1; i <= experiments; i++)
              {
            //pick a random line y = a * x + b
            double x1 = rnd.NextDouble(), y1 = rnd.NextDouble(), x2 = rnd.NextDouble(), y2 = rnd.NextDouble();
            var Wf = new DenseVector(3);
            Wf[0] = 1;
            Wf[1] = (y1 - y2) / (x1 * y2 - y1 * x2);
            Wf[2] = (x2 - x1) / (x1 * y2 - y1 * x2);
            Func<MathNet.Numerics.LinearAlgebra.Generic.Vector<double>, int> f = x => Wf.DotProduct(x) >= 0 ? 1 : -1;

            //generate training set of N random points
            var X = new DenseMatrix(points, 3);
            do
              for (int j = 0; j < points; j++)
              {
            X[j, 0] = 1;
            X[j, 1] = rnd.NextDouble() * 2 - 1;
            X[j, 2] = rnd.NextDouble() * 2 - 1;
              }
            while (Enumerable.Range(0, X.RowCount).All(j => f(X.Row(0)) == f(X.Row(j))));

            var W = new DenseVector(3);
            Func<MathNet.Numerics.LinearAlgebra.Generic.Vector<double>, int> h = x => W.DotProduct(x) >= 0 ? 1 : -1;

            //run Perceptron
            int k = 1;
            while (Enumerable.Range(0, points).Any(j => h(X.Row(j)) != f(X.Row(j))))
            {
              //find all misclasified points
              int[] M = Enumerable.Range(0, points).Where(j => h(X.Row(j)) != f(X.Row(j))).ToArray();
              int m = M[rnd.Next(0, M.Length)];

              int sign = f(X.Row(m));
              W[0] += sign;
              W[1] += sign * X[m, 1];
              W[2] += sign * X[m, 2];
              k++;
            }

            //calculate P[f(Xtest) != h(Xtest)]
            DenseVector Xtest = new DenseVector(3);
            Xtest[0] = 1;
            int matches = 0;
            for (int j = 0; j < TEST_POINTS; j++)
            {
              Xtest[1] = rnd.NextDouble() * 2 - 1;
              Xtest[2] = rnd.NextDouble() * 2 - 1;
              if (f(Xtest) == h(Xtest)) matches++;
            }
            double Ppla = (matches + 0.0) / TEST_POINTS;

            //Run SVM
            var prob = new svm_problem() {
              x = Enumerable.Range(0, points).Select(j =>
            new svm_node[] {
              new svm_node() { index = 0, value = X[j, 1] },
              new svm_node() { index = 1, value = X[j, 2] } }).ToArray(),
              y = Enumerable.Range(0, points).Select(j => (double)f(X.Row(j))).ToArray(),
              l = points };

            var model = svm.svm_train(prob, new svm_parameter()
            {
              svm_type = (int)SvmType.C_SVC,
              kernel_type = (int)KernelType.LINEAR,
              C = 1000000,
              eps = 0.001,
              shrinking = 0
            });

            //calculate P[f(Xtest) != h_svm(Xtest)]
            svm_node[] Xsvm = new svm_node[] {
              new svm_node() { index = 0, value = 1.0 },
              new svm_node() { index = 1, value = 1.0 } };
            matches = 0;

            for (int j = 0; j < TEST_POINTS; j++)
            {
              Xtest[1] = rnd.NextDouble() * 2 - 1;
              Xsvm[0].value = Xtest[1];
              Xtest[2] = rnd.NextDouble() * 2 - 1;
              Xsvm[1].value = Xtest[2];
              if (f(Xtest) == (svm.svm_predict(model, Xsvm) > 0 ? 1 : -1)) matches++;
            }
            double Psvm = (matches + 0.0) / TEST_POINTS;

            svCount += model.l;
            if (Psvm >= Ppla) svmWins++;
              }

              return Tuple.Create((svmWins + 0.0) / experiments, (svCount + 0.0) / experiments);
        }
Пример #12
0
        public void Predict(double[] newPredictData)
        {
            double error = 0;
            int c = 0;

            double[] newNormalizedData = normalizeArray.Process(newPredictData);

            var d = new DenseMatrix(2, newNormalizedData.Length - WindowSize + 1, double.NaN);
            int count = 0;
            for (int i = WindowSize; i < newNormalizedData.Length; i++)
            {
                // calculate based on actual data
                IMLData input = new BasicMLData(WindowSize);
                for (int j = 0; j < input.Count; j++)
                {
                    input.Data[j] = newNormalizedData[(i - WindowSize) + j];
                }

                IMLData output = network.Compute(input);
                double prediction = output.Data[0];


                error += Math.Pow((normalizeArray.Stats.DeNormalize(prediction) - newPredictData[i])/newPredictData[i],
                    2);
                c++;
                d[0, count] = newPredictData[i];
                d[1, count] = normalizeArray.Stats.DeNormalize(prediction);
                count++;
            }

            ///////////////////////////////////////////////////////////////////////////////

            var lastData = new double[WindowSize];
            int count1 = 0;
            for (int i = newNormalizedData.Length - WindowSize; i < newNormalizedData.Length; i++)
            {
                lastData[count1++] = newNormalizedData[i];
            }
            IMLData input1 = new BasicMLData(WindowSize);

            for (int j = 0; j < input1.Count; j++)
            {
                input1.Data[j] = lastData[j];
            }


            IMLData output1 = network.Compute(input1);
            d[1, count] = normalizeArray.Stats.DeNormalize(output1.Data[0]);


            /////////////////////////////////////////////////////////////////////////////////


            error /= c;
            error = Math.Pow(error, .5);
            Console.WriteLine(error);

            OutputData = d.Row(1).ToArray();

            string[] symbols = {"actual", "predicted"};
            Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0),
                QSConstants.DEFAULT_DATA_FILEPATH + write + ".html");

            Console.WriteLine("ST1 Correlation: " +
                              StatisticsExtension.Correlation(
                                  d.Row(0).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn(),
                                  d.Row(1).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn()));
        }
Пример #13
0
        public void Predict()
        {
            double error = 0;
            int c = 0;

            var d = new DenseMatrix(2, _normalizedPredictionData.Length - WindowSize);
            int count = 0;
            for (int i = WindowSize; i < _normalizedPredictionData.Length; i++)
            {
                // calculate based on actual data
                IMLData input = new BasicMLData(WindowSize);
                for (int j = 0; j < input.Count; j++)
                {
                    input.Data[j] = _normalizedPredictionData[(i - WindowSize) + j];
                }

                IMLData output = network.Compute(input);
                double prediction = output.Data[0];


                error += Math.Pow((normalizeArray.Stats.DeNormalize(prediction) - predictionData[i])/predictionData[i],
                    2);
                c++;
                d[0, count] = predictionData[i];
                d[1, count] = normalizeArray.Stats.DeNormalize(prediction);
                count++;
            }

            error /= c;
            error = Math.Pow(error, .5);
            Console.WriteLine(error);

            OutputData = d.Row(1).ToArray();

            string[] symbols = {"actual", "predicted"};
            Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0),
                QSConstants.DEFAULT_DATA_FILEPATH + write + ".html");
        }
Пример #14
0
        public void FinishAndProcess()
        {
            try
            {
                var priceData = new DenseMatrix(symbols.Length, numTicks);

                for (int j = 0; j < symbols.Length; j++)
                {
                    SortedList<DateTime, Tick> d = mktData[j].data.Data;
                    for (int k = 0; k < d.Count; k++)
                    {
                        //if (!symbols[j].Substring(0, 3).Equals("USD")) priceData[j, k] = 1/d.Values[k].BidClose;
                        priceData[j, k] = d.Values[k].BidOpen;
                    }
                }

                Vector<double> price1 = priceData.Row(0);
                Vector<double> price2 = priceData.Row(1);
                //Statistics.ApplyFunction((DenseVector)price1, Math.Log);
                //Statistics.ApplyFunction((DenseVector)price2, Math.Log);

                DenseVector norm1 = price1.ToArray().NormalizeZScore();
                DenseVector norm2 = price2.ToArray().NormalizeZScore();

                var newsym = new string[symbols.Length + 4];
                for (int i = 0; i < symbols.Length; i++) newsym[i] = symbols[i];

                newsym[2] = "spread";
                newsym[3] = "EMA5";
                newsym[4] = "EMA15";
                newsym[5] = "EMA30";


                var m = new DenseMatrix(6, norm1.Count);
                m.SetRow(0, norm1);
                m.SetRow(1, norm2);
                m.SetRow(2, (norm1 - norm2).ToArray().NormalizeZScore());
                m.SetRow(3, EMA.CalcEMA(m.Row(2).ToArray(), 5));
                m.SetRow(4, EMA.CalcEMA(m.Row(2).ToArray(), 15));
                m.SetRow(5, EMA.CalcEMA(m.Row(2).ToArray(), 30));

                string filename = symbols[0].Replace('/', '_') + "-" + symbols[1].Replace('/', '_') + ".html";


                ((DenseVector) m.Row(0)).GenerateSimpleGraph("C:\\Sangar\\result.html");

                Visualize.GenerateMultiSymbolGraph(newsym, m, DateTime.Now.AddSeconds(-60*5*300), new TimeSpan(0, 5, 0),
                    "C:\\Sangar\\" + filename);

                FileUpload.UploadFileToFTP("C:\\Sangar\\" + filename, filename);

                Spread = m[2, m.ColumnCount - 1];

                if (Spread > 2.0 && m[2, m.ColumnCount - 2] <= 2.0)
                    Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Above 2.0", "Test");

                if (Spread < -2.0 && m[2, m.ColumnCount - 2] >= -2.0)
                    Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Below -2.0", "Test");

                //if (m[2, m.ColumnCount - 1] < 0.5 && m[2, m.ColumnCount - 2] >= 0.5)
                //    Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Below 0.5", "Test");

                //if (m[2, m.ColumnCount - 1] > -0.5 && m[2, m.ColumnCount - 2] <= -0.5)
                //    Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Above -0.5", "Test");
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
            }
        }
Пример #15
0
        public void FinishAndProcess()
        {
            try
            {
                var priceData = new DenseMatrix(symbols.Length, numTicks);

                for (int j = 0; j < symbols.Length; j++)
                {
                    SortedList<DateTime, Tick> d = mktData[j].data.Data;
                    for (int k = 0; k < d.Count; k++)
                    {
                        priceData[j, k] = d.Values[k].BidClose;
                    }
                }

                for (int i = 0; i < symbols.Length; i++)
                {
                    for (int j = 0; j < symbols.Length; j++)
                    {
                        double[] pDatai = priceData.Row(i).ToArray();
                        double[] pDataj = priceData.Row(j).ToArray();

                        switch (cType)
                        {
                            case CovarianceType.LogReturn:
                            {
                                pDatai = priceData.Row(i).ToArray().LogRateOfReturn();
                                pDataj = priceData.Row(j).ToArray().LogRateOfReturn();
                                break;
                            }
                            case CovarianceType.RawReturn:
                            {
                                pDatai = priceData.Row(i).ToArray().RawRateOfReturn();
                                pDataj = priceData.Row(j).ToArray().RawRateOfReturn();
                                break;
                            }
                        }
                        correlation[i, j] = StatisticsExtension.Correlation(pDatai, pDataj);
                        covariance[i, j] = StatisticsExtension.Covariance((DenseVector) priceData.Row(i),
                            (DenseVector) priceData.Row(j));
                    }
                }

                Visualize.GenerateHeatMatrix(symbols, correlation, "C:\\Users\\Ethan\\Work\\QuantSysdata.html");
                Console.WriteLine("Finished Generating Correlation Matrix.");
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
            }
        }
Пример #16
0
        /// <summary>
        /// Generate a random n-class classification problem.
        /// </summary>
        /// <param name="nSamples">The number of samples.</param>
        /// <param name="nFeatures">The total number of features. These comprise <paramref name="nInformative"/>
        /// informative features, <paramref name="nRedundant"/> redundant features, <paramref name="nRepeated"/>
        /// dupplicated features and `<paramref name="nFeatures"/>-<paramref name="nInformative"/>-<paramref name="nRedundant"/>-
        /// <paramref name="nRepeated"/>` useless features drawn at random.</param>
        /// <param name="nInformative">The number of informative features. Each class is composed of a number
        /// of gaussian clusters each located around the vertices of a hypercube
        /// in a subspace of dimension <paramref name="nInformative"/>. For each cluster,
        /// informative features are drawn independently from  N(0, 1) and then
        /// randomly linearly combined in order to add covariance. The clusters
        /// are then placed on the vertices of the hypercube.</param>
        /// <param name="nRedundant">The number of redundant features. These features are generated as
        /// random linear combinations of the informative features.</param>
        /// <param name="nRepeated"> The number of dupplicated features, drawn randomly from the informative
        /// and the redundant features.
        /// </param>
        /// <param name="nClasses">The number of classes (or labels) of the classification problem.</param>
        /// <param name="nClustersPerClass">The number of clusters per class.</param>
        /// <param name="weights">The proportions of samples assigned to each class. If None, then
        /// classes are balanced. Note that if `len(weights) == n_classes - 1`,
        /// then the last class weight is automatically inferred.
        /// </param>
        /// <param name="flipY">The fraction of samples whose class are randomly exchanged.</param>
        /// <param name="classSep">The factor multiplying the hypercube dimension.</param>
        /// <param name="hypercube">If True, the clusters are put on the vertices of a hypercube. If
        /// False, the clusters are put on the vertices of a random polytope.</param>
        /// <param name="shift">Shift all features by the specified value. If None, then features
        /// are shifted by a random value drawn in [-class_sep, class_sep].</param>
        /// <param name="scale">Multiply all features by the specified value. If None, then features
        /// are scaled by a random value drawn in [1, 100]. Note that scaling
        /// happens after shifting.
        /// </param>
        /// <param name="shuffle">Shuffle the samples and the features.</param>
        /// <param name="randomState">Random generator.</param>
        /// <returns>array of shape [n_samples]
        /// The integer labels for class membership of each sample.</returns>
        /// <remarks>
        /// The algorithm is adapted from Guyon [1] and was designed to generate
        /// the "Madelon" dataset.
        /// References
        /// ----------
        /// .. [1] I. Guyon, "Design of experiments for the NIPS 2003 variable
        ///   selection benchmark", 2003.
        /// </remarks>
        public static Classification MakeClassification(
            int nSamples = 100,
            int nFeatures = 20,
            int nInformative = 2,
            int nRedundant = 2,
            int nRepeated = 0,
            int nClasses = 2,
            int nClustersPerClass = 2,
            List<double> weights = null,
            double flipY = 0.01,
            double classSep = 1.0,
            bool hypercube = true,
            double? shift = 0.0,
            double? scale = 1.0,
            bool shuffle = true,
            Random randomState = null)
        {
            var generator = randomState ?? new Random();

            // Count features, clusters and samples
            if (nInformative + nRedundant + nRepeated > nFeatures)
            {
                throw new ArgumentException("Number of informative, redundant and repeated " +
                                            "features must sum to less than the number of total" +
                                            " features");
            }

            if (nInformative * nInformative < nClasses * nClustersPerClass)
            {
                throw new ArgumentException(
                    "n_classes * n_clusters_per_class must" +
                    "be smaller or equal 2 ** n_informative");
            }

            if (weights != null && !new[] { nClasses, nClasses - 1 }.Contains(weights.Count))
            {
                throw new ArgumentException("Weights specified but incompatible with number of classes.");
            }

            int nUseless = nFeatures - nInformative - nRedundant - nRepeated;
            int nClusters = nClasses * nClustersPerClass;

            if (weights != null && weights.Count == nClasses - 1)
            {
                weights.Add(1.0 - weights.Sum());
            }

            if (weights == null) 
            {
                weights = Enumerable.Repeat(1.0 / nClasses, nClasses).ToList();
                weights[weights.Count - 1] = 1.0 - weights.Take(weights.Count - 1).Sum();
            }

            var nSamplesPerCluster = new List<int>();

            for (int k = 0; k < nClusters; k++)
            {
                nSamplesPerCluster.Add(
                    (int)(nSamples * weights[k % nClasses] / nClustersPerClass));
            }

            for (int i = 0; i < nSamples - nSamplesPerCluster.Sum(); i++)
            {
                nSamplesPerCluster[i % nClusters] += 1;
            }

            // Intialize X and y
            Matrix x = new DenseMatrix(nSamples, nFeatures);
            int[] y = new int[nSamples];

            // Build the polytope
            Matrix c = new DenseMatrix(1 << nInformative, nInformative);
            for (int i = 0; i < 1 << nInformative; i++)
            {
                var row = new DenseVector(nInformative);
                for (int bitN = 0; bitN < nInformative; bitN++)
                {
                    row[bitN] = (i & (1 << bitN)) == 1 ? classSep : -classSep;
                }

                c.SetRow(i, row);
            }

            if (!hypercube)
            {
                for (int k = 0; k < nClusters; k++)
                {
                    c.SetRow(k, c.Row(k) * generator.NextDouble());
                }

                for (int f = 0; f < nInformative; f++)
                {
                    c.SetColumn(f, c.Column(f) * generator.NextDouble());
                }
            }

            // todo:
            // generator.shuffle(C)

            // Loop over all clusters
            int pos = 0;
            int posEnd = 0;

            for (int k = 0; k < nClusters; k++)
            {
                // Number of samples in cluster k
                int nSamplesK = nSamplesPerCluster[k];

                // Define the range of samples
                pos = posEnd;
                posEnd = pos + nSamplesK;

                // Assign labels
                for (int l = pos; l < posEnd; l++)
                {
                    y[l] = k % nClasses;
                }

                // Draw features at random
                var subMatrix = DenseMatrix.CreateRandom(
                    nSamplesK,
                    nInformative,
                    new Normal { RandomSource = generator });

                x.SetSubMatrix(pos, nSamplesK, 0, nInformative, subMatrix);

                // Multiply by a random matrix to create co-variance of the features
                var uniform = new ContinuousUniform(-1, 1) { RandomSource = generator };
                Matrix a = DenseMatrix.CreateRandom(nInformative, nInformative, uniform);

                x.SetSubMatrix(
                    pos,
                    nSamplesK,
                    0,
                    nInformative,
                    x.SubMatrix(pos, nSamplesK, 0, nInformative) * a);

                // Shift the cluster to a vertice
                var v = x.SubMatrix(pos, nSamplesK, 0, nInformative).AddRowVector(c.Row(k));
                x.SetSubMatrix(pos, nSamplesK, 0, nInformative, v);
            }

            // Create redundant features
            if (nRedundant > 0)
            {
                var uniform = new ContinuousUniform(-1, 1) { RandomSource = generator };
                Matrix b = DenseMatrix.CreateRandom(nInformative, nRedundant, uniform);
                x.SetSubMatrix(
                    0,
                    x.RowCount,
                    nInformative,
                    nRedundant,
                    x.SubMatrix(0, x.RowCount, 0, nInformative) * b);
            }

            // Repeat some features
            if (nRepeated > 0)
            {
                int n = nInformative + nRedundant;
                for (int i = 0; i < nRepeated; i++)
                {
                    int r = (int)((generator.Next(nRepeated) * (n - 1)) + 0.5);
                    x.SetColumn(i, x.Column(r));
                }
            }

            // Fill useless features
            var denseMatrix = DenseMatrix.CreateRandom(nSamples, nUseless, new Normal { RandomSource = generator });
            x.SetSubMatrix(0, nSamples, nFeatures - nUseless, nUseless, denseMatrix);

            // Randomly flip labels
            if (flipY >= 0.0)
            {
                for (int i = 0; i < nSamples; i++)
                {
                    if (generator.NextDouble() < flipY)
                    {
                        y[i] = generator.Next(nClasses);
                    }
                }
            }

            // Randomly shift and scale
            bool constantShift = shift != null;
            bool constantScale = scale != null;

            for (int f = 0; f < nFeatures; f++)
            {
                if (!constantShift)
                {
                    shift = ((2 * generator.NextDouble()) - 1) * classSep;
                }

                if (!constantScale)
                {
                    scale = 1 + (100 * generator.NextDouble());
                }

                x.SetColumn(f, (x.Column(f) + shift.Value) * scale.Value);
            }

            // Randomly permute samples and features
            // todo:
            /*
            if (shuffle)
            {
                X, y = util_shuffle(X, y, random_state=generator)

                indices = np.arange(n_features)
                generator.shuffle(indices)
                X[:, :] = X[:, indices]
            }*/

            return new Classification { X = x, Y = y };
        }
Пример #17
0
        public void Execute()
        {
            var nnSet = new Stage1NeuralNetwork[vectors.Length];

            int trainLength = 2000;
            int validationLength = 500;
            int predictLength = 500;
            int useLength = 3000;

            var totalData = new DenseMatrix(vectors.Length, useLength);
            var outputData = new DenseMatrix(vectors.Length, validationLength - window);

            /////////////////populate the actual price data we want to predict
            var pricingData = new double[useLength];

            for (int i = 2; i < 2 + useLength; i++)
            {
                pricingData[i - 2] = (double) data[useLength + 3 - i, 5];
            }


            double[] returnpricingData = pricingData.RawRateOfReturn();
            for (int i = 0; i < returnpricingData.Length; i++) pricingData[i + 1] = returnpricingData[i];
            pricingData[0] = 0;


            ////////////////////////training and validation////////////////////////
            for (int i = 2; i < 2 + useLength; i++)
            {
                for (int j = 0; j < vectors.Length; j++)
                {
                    totalData[j, i - 2] = (double) data[useLength + 3 - i, vectors[j]];
                }
            }


            for (int j = 0; j < vectors.Length; j++)
            {
                double[] train = totalData.Row(j).ToArray().Take(trainLength).ToArray();
                double[] validate =
                    totalData.Row(j).ToArray().Skip(trainLength).ToArray().Take(validationLength).ToArray();
                nnSet[j] = new Stage1NeuralNetwork(window, cycles, train, validate);
                nnSet[j].Execute(j);
                outputData.SetRow(j, nnSet[j].OutputData);
            }

            var s1 = new Stage2NeuralNetwork(vectors.Length, cycles, outputData,
                pricingData.Skip(trainLength).ToArray().Take(validationLength).ToArray().Skip(window).ToArray());
            s1.Execute();

            //////////////////////////////////////////////////////////////////////////
            //////////////////////////////////prediction/////////////////////////////
            var predictedData = new DenseMatrix(vectors.Length, predictLength - window + 1);

            var lastPredData = new double[vectors.Length];

            for (int j = 0; j < vectors.Length; j++)
            {
                double[] predictData =
                    totalData.Row(j)
                        .ToArray()
                        .Skip(trainLength + validationLength)
                        .ToArray()
                        .Take(predictLength)
                        .ToArray();
                nnSet[j].Predict(predictData);
                predictedData.SetRow(j, nnSet[j].OutputData);
                lastPredData[j] = nnSet[j].NextPrediction;
            }

            s1.Predict(predictedData,
                pricingData.ToArray()
                    .Skip(trainLength + validationLength)
                    .ToArray()
                    .Take(predictLength)
                    .ToArray()
                    .Skip(window)
                    .ToArray());

            correlation = s1.outputCorre;
            RMSE = s1.outputRMSE;
        }
Пример #18
0
        /// <summary>
        /// Run example
        /// </summary>
        public void Run()
        {
            // Format vector output to console
            var formatProvider = (CultureInfo)CultureInfo.InvariantCulture.Clone();
            formatProvider.TextInfo.ListSeparator = " ";

            // Create new empty square matrix
            var matrix = new DenseMatrix(10);
            Console.WriteLine(@"Empty matrix");
            Console.WriteLine(matrix.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 1. Fill matrix by data using indexer []
            var k = 0;
            for (var i = 0; i < matrix.RowCount; i++)
            {
                for (var j = 0; j < matrix.ColumnCount; j++)
                {
                    matrix[i, j] = k++;
                }
            }

            Console.WriteLine(@"1. Fill matrix by data using indexer []");
            Console.WriteLine(matrix.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 2. Fill matrix by data using At. The element is set without range checking.
            for (var i = 0; i < matrix.RowCount; i++)
            {
                for (var j = 0; j < matrix.ColumnCount; j++)
                {
                    matrix.At(i, j, k--);
                }
            }

            Console.WriteLine(@"2. Fill matrix by data using At");
            Console.WriteLine(matrix.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 3. Clone matrix
            var clone = matrix.Clone();
            Console.WriteLine(@"3. Clone matrix");
            Console.WriteLine(clone.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 4. Clear matrix
            clone.Clear();
            Console.WriteLine(@"4. Clear matrix");
            Console.WriteLine(clone.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 5. Copy matrix into another matrix
            matrix.CopyTo(clone);
            Console.WriteLine(@"5. Copy matrix into another matrix");
            Console.WriteLine(clone.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 6. Get submatrix into another matrix
            var submatrix = matrix.SubMatrix(2, 2, 3, 3);
            Console.WriteLine(@"6. Copy submatrix into another matrix");
            Console.WriteLine(submatrix.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 7. Get part of the row as vector. In this example: get 4 elements from row 5 starting from column 3
            var row = matrix.Row(5, 3, 4);
            Console.WriteLine(@"7. Get part of the row as vector");
            Console.WriteLine(row.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 8. Get part of the column as vector. In this example: get 3 elements from column 2 starting from row 6
            var column = matrix.Column(2, 6, 3);
            Console.WriteLine(@"8. Get part of the column as vector");
            Console.WriteLine(column.ToString("#0.00\t", formatProvider));
            Console.WriteLine();

            // 9. Get columns using column enumerator. If you need all columns you may use ColumnEnumerator without parameters
            Console.WriteLine(@"9. Get columns using column enumerator");
            foreach (var keyValuePair in matrix.ColumnEnumerator(2, 4))
            {
                Console.WriteLine(@"Column {0}: {1}", keyValuePair.Item1, keyValuePair.Item2.ToString("#0.00\t", formatProvider));
            }

            Console.WriteLine();

            // 10. Get rows using row enumerator. If you need all rows you may use RowEnumerator without parameters
            Console.WriteLine(@"10. Get rows using row enumerator");
            foreach (var keyValuePair in matrix.RowEnumerator(4, 3))
            {
                Console.WriteLine(@"Row {0}: {1}", keyValuePair.Item1, keyValuePair.Item2.ToString("#0.00\t", formatProvider));
            }

            Console.WriteLine();

            // 11. Convert matrix into multidimensional array
            var data = matrix.ToArray();
            Console.WriteLine(@"11. Convert matrix into multidimensional array");
            for (var i = 0; i < data.GetLongLength(0); i++)
            {
                for (var j = 0; j < data.GetLongLength(1); j++)
                {
                    Console.Write(data[i, j].ToString("#0.00\t"));
                }

                Console.WriteLine();
            }

            Console.WriteLine();

            // 12. Convert matrix into row-wise array
            var rowwise = matrix.ToRowWiseArray();
            Console.WriteLine(@"12. Convert matrix into row-wise array");
            for (var i = 0; i < matrix.RowCount; i++)
            {
                for (var j = 0; j < matrix.ColumnCount; j++)
                {
                    Console.Write(rowwise[(i * matrix.ColumnCount) + j].ToString("#0.00\t"));
                }

                Console.WriteLine();
            }

            Console.WriteLine();

            // 13. Convert matrix into column-wise array
            var columnise = matrix.ToColumnWiseArray();
            Console.WriteLine(@"13. Convert matrix into column-wise array");
            for (var i = 0; i < matrix.RowCount; i++)
            {
                for (var j = 0; j < matrix.ColumnCount; j++)
                {
                    Console.Write(columnise[(j * matrix.RowCount) + i].ToString("#0.00\t"));
                }

                Console.WriteLine();
            }

            Console.WriteLine();

            // 14. Get matrix diagonal as vector
            var diagonal = matrix.Diagonal();
            Console.WriteLine(@"14. Get matrix diagonal as vector");
            Console.WriteLine(diagonal.ToString("#0.00\t", formatProvider));
            Console.WriteLine();
        }
Пример #19
0
        private void ReduceCosts()
        {
            ReducedCosts = (DenseMatrix)Costs.Clone();

            for (int i = 0; i < N; i++)
            {
                var min = ReducedCosts.Row(i).Minimum();
                for (int j = 0; j < N; j++)
                {
                    ReducedCosts[i, j] -= min;
                }
            }
            //todo: maybe round result
            for (int j = 0; j < N; j++)
            {
                var min = ReducedCosts.Column(j).Minimum();
                for (int i = 0; i < N; i++)
                {
                    ReducedCosts[i, j] -= min;
                }
            }
            //todo: maybe round result
        }