public DenseMatrix NormalizeData(DenseMatrix data) { var normalizedData = new DenseMatrix(data.RowCount, data.ColumnCount); for (int i = 0; i < data.RowCount; i++) { normalizedData.SetRow(i, normalizeArrayInput[i].Process(data.Row(i).ToArray())); } return normalizedData; }
private static double Error(DenseVector weights, DenseMatrix xM, DenseVector yV) { Debug.Assert(xM.RowCount == yV.Count); Debug.Assert(xM.ColumnCount == weights.Count); double wrong = 0.0; for (int i = 0; i < xM.RowCount; i++) { var p = xM.Row(i) * weights; if ((p < 0 ? -1 : 1) != yV[i]) wrong += 1; } return wrong / xM.RowCount; }
public void Smooth(ref double[,] inputValues) { // TODO: Using the matrix works, but does a lot of data accesses. Can improve by working out all the data access myself? I might be able to cut down on number of data accesses, but not sure. var inputMatrix = new DenseMatrix(inputValues.GetLength(0), inputValues.GetLength(1)); for (int i = 0; i < inputMatrix.RowCount; i++) { inputMatrix.SetRow(i, Smooth(inputMatrix.Row(i).ToArray())); } for (int i = 0; i < inputMatrix.ColumnCount; i++) { inputMatrix.SetColumn(i, Smooth(inputMatrix.Column(i).ToArray())); } inputValues = inputMatrix.ToArray(); }
public void FinishAndProcess() { var priceData = new DenseMatrix(symbols.Length, numTicks); for (int j = 0; j < symbols.Length; j++) { SortedList<DateTime, Tick> d = mktData[j].data.Data; for (int k = 0; k < d.Count; k++) { priceData[j, k] = d.Values[k].BidClose; } } for (int i = 0; i < symbols.Length; i++) { results[i] = function((DenseVector) priceData.Row(i)); } }
public static Dictionary<int, double> GetTopK2(DenseMatrix userSimilarities, int uidTarget, int K) { Dictionary<int, double> topK = new Dictionary<int, double>(K); Vector<double> uidSimilarities = userSimilarities.Row(uidTarget); double minSimilarity = double.MinValue; int minUid = int.MinValue; foreach (Tuple<int, double> entry in uidSimilarities.EnumerateIndexed(Zeros.AllowSkip)) { int uid = entry.Item1; double similarity = entry.Item2; if (uid == uidTarget) { continue; } // A user is not a neighbor of himself if (topK.Count < K) // Fill the top K list untill it is full { topK[uid] = similarity; if (topK.Count == K) { // Find the least similar neighbor when it is full minUid = topK.Aggregate((l, r) => l.Value < r.Value ? l : r).Key; minSimilarity = topK[minUid]; } } else if (similarity > minSimilarity) { // Replace the least similar neighbor topK.Remove(minUid); // The first time it does nothing as the minUid is not set topK[uid] = similarity; // Find the least similar neighbor minUid = topK.Aggregate((l, r) => l.Value < r.Value ? l : r).Key; minSimilarity = topK[minUid]; } } return topK; }
public static void Process(FXSession session, string symbol1, string symbol2, string timeframe, int length) { HistoricPriceEngine h1 = new HistoricPriceEngine(session); h1.GetLongHistoricPrices(symbol1, timeframe, length); while (!h1.Complete) { Thread.Sleep(100); } HistoricPriceEngine h2 = new HistoricPriceEngine(session); h2.GetLongHistoricPrices(symbol2, timeframe, length); while (!h2.Complete) { Thread.Sleep(100); } //----------------------- var dateTimeList = new SortedList<DateTime, int>(); Quantum q1 = h1.Data; Quantum q2 = h2.Data; var priceData = new DenseMatrix(2, q1.Data.Count); for (int j = 0; j < ((q1.Data.Count <= q2.Data.Count)?q1.Data.Count:q2.Data.Count); j++ ) { dateTimeList.Add(q1.Data.Values[j].Time, 1); priceData[0, j] = q1.Data.Values[j].BidClose; priceData[1, j] = q2.Data.Values[j].BidClose; } Vector<double> price1 = priceData.Row(0); Vector<double> price2 = priceData.Row(1); //Statistics.ApplyFunction((DenseVector)price1, Math.Log); //Statistics.ApplyFunction((DenseVector)price2, Math.Log); DenseVector norm1 = price1.ToArray().NormalizeZScore(); DenseVector norm2 = price2.ToArray().NormalizeZScore(); var newsym = new string[] {symbol1, symbol2, "spread"}; var m = new DenseMatrix(6, norm1.Count); m.SetRow(0, norm1); m.SetRow(1, norm2); m.SetRow(2, (norm1 - norm2).ToArray().NormalizeZScore()); string filename = symbol1.Replace('/', '_') + "-" + symbol2.Replace('/', '_') + ".html"; Visualize.GenerateMultiPaneGraph(newsym, dateTimeList.Keys.ToArray(), m, QSConstants.DEFAULT_DATA_FILEPATH + filename, new ChartOption[]{new ChartOption(), new ChartOption(){Layover = true, YPosition = 0}, new ChartOption(){YPosition = 1} }, null, filename + ".json"); FileUpload.UploadFileToFTP(QSConstants.DEFAULT_DATA_FILEPATH + filename, filename); FileUpload.UploadFileToFTP(QSConstants.DEFAULT_DATA_FILEPATH + filename + ".json", filename + ".json"); double Spread = m[2, m.ColumnCount - 1]; if (Spread > 2.0 && m[2, m.ColumnCount - 2] <= 2.0) Emailer.SendEmail(symbol1 + "-" + symbol2 + " Spread Above 2.0", "Test"); if (Spread < -2.0 && m[2, m.ColumnCount - 2] >= -2.0) Emailer.SendEmail(symbol1 + "-" + symbol2 + " Spread Below -2.0", "Test"); }
public void Train(DenseMatrix X, DenseVector d, DenseVector Kd) { int R = X.RowCount; int N = X.ColumnCount; int U = 0; //the number of neurons in the structure var c = new DenseMatrix(R, 1); var sigma = new DenseMatrix(R, 1); var Q = new DenseMatrix((R + 1), (R + 1)); var O = new DenseMatrix(1, (R + 1)); var pT_n = new DenseMatrix((R + 1), 1); double maxPhi = 0; int maxIndex; var Psi = new DenseMatrix(N, 1); Console.WriteLine("Running..."); //for each observation n in X for (int i = 0; i < N; i++) { Console.WriteLine(100*(i/(double) N) + "%"); var x = new DenseVector(R); X.Column(i, x); //if there are neurons in structure, //update structure recursively. if (U == 0) { c = (DenseMatrix) x.ToColumnMatrix(); sigma = new DenseMatrix(R, 1, SigmaZero); U = 1; Psi = CalculatePsi(X, c, sigma); UpdateStructure(X, Psi, d, ref Q, ref O); pT_n = (DenseMatrix) (CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) Psi.Row(i).ToRowMatrix())) .Transpose(); } else { StructureRecurse(X, Psi, d, i, ref Q, ref O, ref pT_n); } bool KeepSpinning = true; while (KeepSpinning) { //Calculate the error and if-part criteria double ee = pT_n.Multiply(O)[0, 0]; double approximationError = Math.Abs(d[i] - ee); DenseVector Phi; double SumPhi; CalculatePhi(x, c, sigma, out Phi, out SumPhi); maxPhi = Phi.Maximum(); maxIndex = Phi.MaximumIndex(); if (approximationError > delta) { if (maxPhi < threshold) { var tempSigma = new DenseVector(R); sigma.Column(maxIndex, tempSigma); double minSigma = tempSigma.Minimum(); int minIndex = tempSigma.MinimumIndex(); sigma[minIndex, maxIndex] = k_sigma*minSigma; Psi = CalculatePsi(X, c, sigma); UpdateStructure(X, Psi, d, ref Q, ref O); var psi = new DenseVector(Psi.ColumnCount); Psi.Row(i, psi); pT_n = (DenseMatrix) CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix()) .Transpose(); } else { //add a new neuron and update strucutre double distance = 0; var cTemp = new DenseVector(R); var sigmaTemp = new DenseVector(R); //foreach input variable for (int j = 0; j < R; j++) { distance = Math.Abs(x[j] - c[j, 0]); int distanceIndex = 0; //foreach neuron past 1 for (int k = 1; k < U; k++) { if ((Math.Abs(x[j] - c[j, k])) < distance) { distanceIndex = k; distance = Math.Abs(x[j] - c[j, k]); } } if (distance < Kd[j]) { cTemp[j] = c[j, distanceIndex]; sigmaTemp[j] = sigma[j, distanceIndex]; } else { cTemp[j] = x[j]; sigmaTemp[j] = distance; } } //end foreach c = (DenseMatrix) c.InsertColumn(c.ColumnCount - 1, cTemp); sigma = (DenseMatrix) sigma.InsertColumn(sigma.ColumnCount - 1, sigmaTemp); Psi = CalculatePsi(X, c, sigma); UpdateStructure(X, Psi, d, ref Q, ref O); U++; KeepSpinning = false; } } else { if (maxPhi < threshold) { var tempSigma = new DenseVector(R); sigma.Column(maxIndex, tempSigma); double minSigma = tempSigma.Minimum(); int minIndex = tempSigma.MinimumIndex(); sigma[minIndex, maxIndex] = k_sigma*minSigma; Psi = CalculatePsi(X, c, sigma); UpdateStructure(X, Psi, d, ref Q, ref O); var psi = new DenseVector(Psi.ColumnCount); Psi.Row(i, psi); pT_n = (DenseMatrix) CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix()) .Transpose(); } else { KeepSpinning = false; } } } } out_C = c; out_O = O; out_Sigma = sigma; Console.WriteLine("Done."); }
public void StructureRecurse(DenseMatrix X, DenseMatrix Psi, DenseVector d, int n, ref DenseMatrix Q, ref DenseMatrix O, ref DenseMatrix pT_n) { //O = O(t-1) O_enxt = O(t) //o should be a column vector ( in matrix form) var x = new DenseVector(X.RowCount); var psi = new DenseVector(Psi.ColumnCount); X.Column(n, x); Psi.Row(n, psi); DenseMatrix p_n = CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix()); pT_n = (DenseMatrix) p_n.Transpose(); double ee = Math.Abs(d[n] - (pT_n.Multiply(O))[0, 0]); double temp = 1 + (pT_n.Multiply(Q)).Multiply(p_n)[0, 0]; double ae = Math.Abs(ee/temp); if (ee >= ae) { var L = (DenseMatrix) Q.Multiply(p_n).Multiply(1/temp); Q = (DenseMatrix) ((DenseMatrix.Identity(Q.RowCount).Subtract(L.Multiply(pT_n))).Multiply(Q)); O = (DenseMatrix) O.Add(L*ee); } else { Q = (DenseMatrix) DenseMatrix.Identity(Q.RowCount).Multiply(Q); } }
public void Predict(DenseMatrix newPredictData, double[] newPredictPrices) { double error = 0; int c = 0; var newNormalizedPredictData = new DenseMatrix(newPredictData.RowCount, newPredictData.ColumnCount, double.NaN); for (int i = 0; i < newPredictData.RowCount; i++) { newNormalizedPredictData.SetRow(i, normalizeArrayInput[i].Process(newPredictData.Row(i).ToArray())); } double[] normalizedPrices = normalizeArrayOutput.Process(newPredictPrices); var d = new DenseMatrix(2, normalizedPrices.Length + 1, double.NaN); int count = 0; for (int i = 0; i < normalizedPrices.Length; i++) { // calculate based on actual data IMLData input = new BasicMLData(inputs); for (int j = 0; j < input.Count; j++) { input.Data[j] = newNormalizedPredictData[j, i]; } IMLData output = network.Compute(input); double prediction = output.Data[0]; error += Math.Pow( (normalizeArrayOutput.Stats.DeNormalize(prediction) - newPredictPrices[i])/newPredictPrices[i], 2); c++; d[0, count] = newPredictPrices[i]; d[1, count] = normalizeArrayOutput.Stats.DeNormalize(prediction); count++; } ///////////////////////////////////////////////////////////////// IMLData input1 = new BasicMLData(inputs); for (int j = 0; j < input1.Count; j++) { input1.Data[j] = newNormalizedPredictData[j, newNormalizedPredictData.ColumnCount - 1]; } IMLData output1 = network.Compute(input1); d[1, count] = normalizeArrayOutput.Stats.DeNormalize(output1.Data[0]); ///////////////////////////////////////////////////////////////// error /= c; error = Math.Pow(error, .5); Console.WriteLine(error); string[] symbols = {"actual", "predicted"}; Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0), "C:\\Sangar\\resultfinal.html"); outputCorre = StatisticsExtension.Correlation(d.Row(0).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn(), d.Row(1).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn()); Console.WriteLine("ST2 Correlation: " + outputCorre); outputRMSE = error; Console.WriteLine("Predicted return for D+1:" + (d[1, d.ColumnCount - 1] - d[1, d.ColumnCount - 2])/d[1, d.ColumnCount - 2]*100 + " percent"); }
public Krige(DenseMatrix xx, DenseMatrix yy, Powvargram vg) { // Initializer x = xx; vgram = vg; n = xx.RowCount; d = xx.ColumnCount; dstar = new DenseMatrix(n + 1, 1); vstar = new DenseMatrix(n + 1, 1); v = new DenseMatrix(n + 1, n + 1); y = new DenseMatrix(n + 1, n); // Building matrix V,Y for (int i = 0; i < n; i++) { y[i, 0] = yy[i, 0]; for (int j = i; j < n; j++) { double vvv = vgram.GetV((x.Row(i) - x.Row(j)).Norm(2)); v[i, j] = vvv; v[j, i] = vvv; } v[i, n] = 1.0; v[n, i] = 1.0; } v[n, n] = 0.0; y[n, 0] = 0.0; vi = v.Inverse(); yvi = vi.Transpose() * y; }
static Tuple<double, double> RunPLAvsSVM(int experiments, int points) { const int TEST_POINTS = 10000; Random rnd = new Random(); long svmWins = 0, svCount = 0; for (int i = 1; i <= experiments; i++) { //pick a random line y = a * x + b double x1 = rnd.NextDouble(), y1 = rnd.NextDouble(), x2 = rnd.NextDouble(), y2 = rnd.NextDouble(); var Wf = new DenseVector(3); Wf[0] = 1; Wf[1] = (y1 - y2) / (x1 * y2 - y1 * x2); Wf[2] = (x2 - x1) / (x1 * y2 - y1 * x2); Func<MathNet.Numerics.LinearAlgebra.Generic.Vector<double>, int> f = x => Wf.DotProduct(x) >= 0 ? 1 : -1; //generate training set of N random points var X = new DenseMatrix(points, 3); do for (int j = 0; j < points; j++) { X[j, 0] = 1; X[j, 1] = rnd.NextDouble() * 2 - 1; X[j, 2] = rnd.NextDouble() * 2 - 1; } while (Enumerable.Range(0, X.RowCount).All(j => f(X.Row(0)) == f(X.Row(j)))); var W = new DenseVector(3); Func<MathNet.Numerics.LinearAlgebra.Generic.Vector<double>, int> h = x => W.DotProduct(x) >= 0 ? 1 : -1; //run Perceptron int k = 1; while (Enumerable.Range(0, points).Any(j => h(X.Row(j)) != f(X.Row(j)))) { //find all misclasified points int[] M = Enumerable.Range(0, points).Where(j => h(X.Row(j)) != f(X.Row(j))).ToArray(); int m = M[rnd.Next(0, M.Length)]; int sign = f(X.Row(m)); W[0] += sign; W[1] += sign * X[m, 1]; W[2] += sign * X[m, 2]; k++; } //calculate P[f(Xtest) != h(Xtest)] DenseVector Xtest = new DenseVector(3); Xtest[0] = 1; int matches = 0; for (int j = 0; j < TEST_POINTS; j++) { Xtest[1] = rnd.NextDouble() * 2 - 1; Xtest[2] = rnd.NextDouble() * 2 - 1; if (f(Xtest) == h(Xtest)) matches++; } double Ppla = (matches + 0.0) / TEST_POINTS; //Run SVM var prob = new svm_problem() { x = Enumerable.Range(0, points).Select(j => new svm_node[] { new svm_node() { index = 0, value = X[j, 1] }, new svm_node() { index = 1, value = X[j, 2] } }).ToArray(), y = Enumerable.Range(0, points).Select(j => (double)f(X.Row(j))).ToArray(), l = points }; var model = svm.svm_train(prob, new svm_parameter() { svm_type = (int)SvmType.C_SVC, kernel_type = (int)KernelType.LINEAR, C = 1000000, eps = 0.001, shrinking = 0 }); //calculate P[f(Xtest) != h_svm(Xtest)] svm_node[] Xsvm = new svm_node[] { new svm_node() { index = 0, value = 1.0 }, new svm_node() { index = 1, value = 1.0 } }; matches = 0; for (int j = 0; j < TEST_POINTS; j++) { Xtest[1] = rnd.NextDouble() * 2 - 1; Xsvm[0].value = Xtest[1]; Xtest[2] = rnd.NextDouble() * 2 - 1; Xsvm[1].value = Xtest[2]; if (f(Xtest) == (svm.svm_predict(model, Xsvm) > 0 ? 1 : -1)) matches++; } double Psvm = (matches + 0.0) / TEST_POINTS; svCount += model.l; if (Psvm >= Ppla) svmWins++; } return Tuple.Create((svmWins + 0.0) / experiments, (svCount + 0.0) / experiments); }
public void Predict(double[] newPredictData) { double error = 0; int c = 0; double[] newNormalizedData = normalizeArray.Process(newPredictData); var d = new DenseMatrix(2, newNormalizedData.Length - WindowSize + 1, double.NaN); int count = 0; for (int i = WindowSize; i < newNormalizedData.Length; i++) { // calculate based on actual data IMLData input = new BasicMLData(WindowSize); for (int j = 0; j < input.Count; j++) { input.Data[j] = newNormalizedData[(i - WindowSize) + j]; } IMLData output = network.Compute(input); double prediction = output.Data[0]; error += Math.Pow((normalizeArray.Stats.DeNormalize(prediction) - newPredictData[i])/newPredictData[i], 2); c++; d[0, count] = newPredictData[i]; d[1, count] = normalizeArray.Stats.DeNormalize(prediction); count++; } /////////////////////////////////////////////////////////////////////////////// var lastData = new double[WindowSize]; int count1 = 0; for (int i = newNormalizedData.Length - WindowSize; i < newNormalizedData.Length; i++) { lastData[count1++] = newNormalizedData[i]; } IMLData input1 = new BasicMLData(WindowSize); for (int j = 0; j < input1.Count; j++) { input1.Data[j] = lastData[j]; } IMLData output1 = network.Compute(input1); d[1, count] = normalizeArray.Stats.DeNormalize(output1.Data[0]); ///////////////////////////////////////////////////////////////////////////////// error /= c; error = Math.Pow(error, .5); Console.WriteLine(error); OutputData = d.Row(1).ToArray(); string[] symbols = {"actual", "predicted"}; Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0), QSConstants.DEFAULT_DATA_FILEPATH + write + ".html"); Console.WriteLine("ST1 Correlation: " + StatisticsExtension.Correlation( d.Row(0).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn(), d.Row(1).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn())); }
public void Predict() { double error = 0; int c = 0; var d = new DenseMatrix(2, _normalizedPredictionData.Length - WindowSize); int count = 0; for (int i = WindowSize; i < _normalizedPredictionData.Length; i++) { // calculate based on actual data IMLData input = new BasicMLData(WindowSize); for (int j = 0; j < input.Count; j++) { input.Data[j] = _normalizedPredictionData[(i - WindowSize) + j]; } IMLData output = network.Compute(input); double prediction = output.Data[0]; error += Math.Pow((normalizeArray.Stats.DeNormalize(prediction) - predictionData[i])/predictionData[i], 2); c++; d[0, count] = predictionData[i]; d[1, count] = normalizeArray.Stats.DeNormalize(prediction); count++; } error /= c; error = Math.Pow(error, .5); Console.WriteLine(error); OutputData = d.Row(1).ToArray(); string[] symbols = {"actual", "predicted"}; Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0), QSConstants.DEFAULT_DATA_FILEPATH + write + ".html"); }
public void FinishAndProcess() { try { var priceData = new DenseMatrix(symbols.Length, numTicks); for (int j = 0; j < symbols.Length; j++) { SortedList<DateTime, Tick> d = mktData[j].data.Data; for (int k = 0; k < d.Count; k++) { //if (!symbols[j].Substring(0, 3).Equals("USD")) priceData[j, k] = 1/d.Values[k].BidClose; priceData[j, k] = d.Values[k].BidOpen; } } Vector<double> price1 = priceData.Row(0); Vector<double> price2 = priceData.Row(1); //Statistics.ApplyFunction((DenseVector)price1, Math.Log); //Statistics.ApplyFunction((DenseVector)price2, Math.Log); DenseVector norm1 = price1.ToArray().NormalizeZScore(); DenseVector norm2 = price2.ToArray().NormalizeZScore(); var newsym = new string[symbols.Length + 4]; for (int i = 0; i < symbols.Length; i++) newsym[i] = symbols[i]; newsym[2] = "spread"; newsym[3] = "EMA5"; newsym[4] = "EMA15"; newsym[5] = "EMA30"; var m = new DenseMatrix(6, norm1.Count); m.SetRow(0, norm1); m.SetRow(1, norm2); m.SetRow(2, (norm1 - norm2).ToArray().NormalizeZScore()); m.SetRow(3, EMA.CalcEMA(m.Row(2).ToArray(), 5)); m.SetRow(4, EMA.CalcEMA(m.Row(2).ToArray(), 15)); m.SetRow(5, EMA.CalcEMA(m.Row(2).ToArray(), 30)); string filename = symbols[0].Replace('/', '_') + "-" + symbols[1].Replace('/', '_') + ".html"; ((DenseVector) m.Row(0)).GenerateSimpleGraph("C:\\Sangar\\result.html"); Visualize.GenerateMultiSymbolGraph(newsym, m, DateTime.Now.AddSeconds(-60*5*300), new TimeSpan(0, 5, 0), "C:\\Sangar\\" + filename); FileUpload.UploadFileToFTP("C:\\Sangar\\" + filename, filename); Spread = m[2, m.ColumnCount - 1]; if (Spread > 2.0 && m[2, m.ColumnCount - 2] <= 2.0) Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Above 2.0", "Test"); if (Spread < -2.0 && m[2, m.ColumnCount - 2] >= -2.0) Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Below -2.0", "Test"); //if (m[2, m.ColumnCount - 1] < 0.5 && m[2, m.ColumnCount - 2] >= 0.5) // Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Below 0.5", "Test"); //if (m[2, m.ColumnCount - 1] > -0.5 && m[2, m.ColumnCount - 2] <= -0.5) // Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Above -0.5", "Test"); } catch (Exception e) { Console.WriteLine(e.Message); } }
public void FinishAndProcess() { try { var priceData = new DenseMatrix(symbols.Length, numTicks); for (int j = 0; j < symbols.Length; j++) { SortedList<DateTime, Tick> d = mktData[j].data.Data; for (int k = 0; k < d.Count; k++) { priceData[j, k] = d.Values[k].BidClose; } } for (int i = 0; i < symbols.Length; i++) { for (int j = 0; j < symbols.Length; j++) { double[] pDatai = priceData.Row(i).ToArray(); double[] pDataj = priceData.Row(j).ToArray(); switch (cType) { case CovarianceType.LogReturn: { pDatai = priceData.Row(i).ToArray().LogRateOfReturn(); pDataj = priceData.Row(j).ToArray().LogRateOfReturn(); break; } case CovarianceType.RawReturn: { pDatai = priceData.Row(i).ToArray().RawRateOfReturn(); pDataj = priceData.Row(j).ToArray().RawRateOfReturn(); break; } } correlation[i, j] = StatisticsExtension.Correlation(pDatai, pDataj); covariance[i, j] = StatisticsExtension.Covariance((DenseVector) priceData.Row(i), (DenseVector) priceData.Row(j)); } } Visualize.GenerateHeatMatrix(symbols, correlation, "C:\\Users\\Ethan\\Work\\QuantSysdata.html"); Console.WriteLine("Finished Generating Correlation Matrix."); } catch (Exception e) { Console.WriteLine(e.Message); } }
/// <summary> /// Generate a random n-class classification problem. /// </summary> /// <param name="nSamples">The number of samples.</param> /// <param name="nFeatures">The total number of features. These comprise <paramref name="nInformative"/> /// informative features, <paramref name="nRedundant"/> redundant features, <paramref name="nRepeated"/> /// dupplicated features and `<paramref name="nFeatures"/>-<paramref name="nInformative"/>-<paramref name="nRedundant"/>- /// <paramref name="nRepeated"/>` useless features drawn at random.</param> /// <param name="nInformative">The number of informative features. Each class is composed of a number /// of gaussian clusters each located around the vertices of a hypercube /// in a subspace of dimension <paramref name="nInformative"/>. For each cluster, /// informative features are drawn independently from N(0, 1) and then /// randomly linearly combined in order to add covariance. The clusters /// are then placed on the vertices of the hypercube.</param> /// <param name="nRedundant">The number of redundant features. These features are generated as /// random linear combinations of the informative features.</param> /// <param name="nRepeated"> The number of dupplicated features, drawn randomly from the informative /// and the redundant features. /// </param> /// <param name="nClasses">The number of classes (or labels) of the classification problem.</param> /// <param name="nClustersPerClass">The number of clusters per class.</param> /// <param name="weights">The proportions of samples assigned to each class. If None, then /// classes are balanced. Note that if `len(weights) == n_classes - 1`, /// then the last class weight is automatically inferred. /// </param> /// <param name="flipY">The fraction of samples whose class are randomly exchanged.</param> /// <param name="classSep">The factor multiplying the hypercube dimension.</param> /// <param name="hypercube">If True, the clusters are put on the vertices of a hypercube. If /// False, the clusters are put on the vertices of a random polytope.</param> /// <param name="shift">Shift all features by the specified value. If None, then features /// are shifted by a random value drawn in [-class_sep, class_sep].</param> /// <param name="scale">Multiply all features by the specified value. If None, then features /// are scaled by a random value drawn in [1, 100]. Note that scaling /// happens after shifting. /// </param> /// <param name="shuffle">Shuffle the samples and the features.</param> /// <param name="randomState">Random generator.</param> /// <returns>array of shape [n_samples] /// The integer labels for class membership of each sample.</returns> /// <remarks> /// The algorithm is adapted from Guyon [1] and was designed to generate /// the "Madelon" dataset. /// References /// ---------- /// .. [1] I. Guyon, "Design of experiments for the NIPS 2003 variable /// selection benchmark", 2003. /// </remarks> public static Classification MakeClassification( int nSamples = 100, int nFeatures = 20, int nInformative = 2, int nRedundant = 2, int nRepeated = 0, int nClasses = 2, int nClustersPerClass = 2, List<double> weights = null, double flipY = 0.01, double classSep = 1.0, bool hypercube = true, double? shift = 0.0, double? scale = 1.0, bool shuffle = true, Random randomState = null) { var generator = randomState ?? new Random(); // Count features, clusters and samples if (nInformative + nRedundant + nRepeated > nFeatures) { throw new ArgumentException("Number of informative, redundant and repeated " + "features must sum to less than the number of total" + " features"); } if (nInformative * nInformative < nClasses * nClustersPerClass) { throw new ArgumentException( "n_classes * n_clusters_per_class must" + "be smaller or equal 2 ** n_informative"); } if (weights != null && !new[] { nClasses, nClasses - 1 }.Contains(weights.Count)) { throw new ArgumentException("Weights specified but incompatible with number of classes."); } int nUseless = nFeatures - nInformative - nRedundant - nRepeated; int nClusters = nClasses * nClustersPerClass; if (weights != null && weights.Count == nClasses - 1) { weights.Add(1.0 - weights.Sum()); } if (weights == null) { weights = Enumerable.Repeat(1.0 / nClasses, nClasses).ToList(); weights[weights.Count - 1] = 1.0 - weights.Take(weights.Count - 1).Sum(); } var nSamplesPerCluster = new List<int>(); for (int k = 0; k < nClusters; k++) { nSamplesPerCluster.Add( (int)(nSamples * weights[k % nClasses] / nClustersPerClass)); } for (int i = 0; i < nSamples - nSamplesPerCluster.Sum(); i++) { nSamplesPerCluster[i % nClusters] += 1; } // Intialize X and y Matrix x = new DenseMatrix(nSamples, nFeatures); int[] y = new int[nSamples]; // Build the polytope Matrix c = new DenseMatrix(1 << nInformative, nInformative); for (int i = 0; i < 1 << nInformative; i++) { var row = new DenseVector(nInformative); for (int bitN = 0; bitN < nInformative; bitN++) { row[bitN] = (i & (1 << bitN)) == 1 ? classSep : -classSep; } c.SetRow(i, row); } if (!hypercube) { for (int k = 0; k < nClusters; k++) { c.SetRow(k, c.Row(k) * generator.NextDouble()); } for (int f = 0; f < nInformative; f++) { c.SetColumn(f, c.Column(f) * generator.NextDouble()); } } // todo: // generator.shuffle(C) // Loop over all clusters int pos = 0; int posEnd = 0; for (int k = 0; k < nClusters; k++) { // Number of samples in cluster k int nSamplesK = nSamplesPerCluster[k]; // Define the range of samples pos = posEnd; posEnd = pos + nSamplesK; // Assign labels for (int l = pos; l < posEnd; l++) { y[l] = k % nClasses; } // Draw features at random var subMatrix = DenseMatrix.CreateRandom( nSamplesK, nInformative, new Normal { RandomSource = generator }); x.SetSubMatrix(pos, nSamplesK, 0, nInformative, subMatrix); // Multiply by a random matrix to create co-variance of the features var uniform = new ContinuousUniform(-1, 1) { RandomSource = generator }; Matrix a = DenseMatrix.CreateRandom(nInformative, nInformative, uniform); x.SetSubMatrix( pos, nSamplesK, 0, nInformative, x.SubMatrix(pos, nSamplesK, 0, nInformative) * a); // Shift the cluster to a vertice var v = x.SubMatrix(pos, nSamplesK, 0, nInformative).AddRowVector(c.Row(k)); x.SetSubMatrix(pos, nSamplesK, 0, nInformative, v); } // Create redundant features if (nRedundant > 0) { var uniform = new ContinuousUniform(-1, 1) { RandomSource = generator }; Matrix b = DenseMatrix.CreateRandom(nInformative, nRedundant, uniform); x.SetSubMatrix( 0, x.RowCount, nInformative, nRedundant, x.SubMatrix(0, x.RowCount, 0, nInformative) * b); } // Repeat some features if (nRepeated > 0) { int n = nInformative + nRedundant; for (int i = 0; i < nRepeated; i++) { int r = (int)((generator.Next(nRepeated) * (n - 1)) + 0.5); x.SetColumn(i, x.Column(r)); } } // Fill useless features var denseMatrix = DenseMatrix.CreateRandom(nSamples, nUseless, new Normal { RandomSource = generator }); x.SetSubMatrix(0, nSamples, nFeatures - nUseless, nUseless, denseMatrix); // Randomly flip labels if (flipY >= 0.0) { for (int i = 0; i < nSamples; i++) { if (generator.NextDouble() < flipY) { y[i] = generator.Next(nClasses); } } } // Randomly shift and scale bool constantShift = shift != null; bool constantScale = scale != null; for (int f = 0; f < nFeatures; f++) { if (!constantShift) { shift = ((2 * generator.NextDouble()) - 1) * classSep; } if (!constantScale) { scale = 1 + (100 * generator.NextDouble()); } x.SetColumn(f, (x.Column(f) + shift.Value) * scale.Value); } // Randomly permute samples and features // todo: /* if (shuffle) { X, y = util_shuffle(X, y, random_state=generator) indices = np.arange(n_features) generator.shuffle(indices) X[:, :] = X[:, indices] }*/ return new Classification { X = x, Y = y }; }
public void Execute() { var nnSet = new Stage1NeuralNetwork[vectors.Length]; int trainLength = 2000; int validationLength = 500; int predictLength = 500; int useLength = 3000; var totalData = new DenseMatrix(vectors.Length, useLength); var outputData = new DenseMatrix(vectors.Length, validationLength - window); /////////////////populate the actual price data we want to predict var pricingData = new double[useLength]; for (int i = 2; i < 2 + useLength; i++) { pricingData[i - 2] = (double) data[useLength + 3 - i, 5]; } double[] returnpricingData = pricingData.RawRateOfReturn(); for (int i = 0; i < returnpricingData.Length; i++) pricingData[i + 1] = returnpricingData[i]; pricingData[0] = 0; ////////////////////////training and validation//////////////////////// for (int i = 2; i < 2 + useLength; i++) { for (int j = 0; j < vectors.Length; j++) { totalData[j, i - 2] = (double) data[useLength + 3 - i, vectors[j]]; } } for (int j = 0; j < vectors.Length; j++) { double[] train = totalData.Row(j).ToArray().Take(trainLength).ToArray(); double[] validate = totalData.Row(j).ToArray().Skip(trainLength).ToArray().Take(validationLength).ToArray(); nnSet[j] = new Stage1NeuralNetwork(window, cycles, train, validate); nnSet[j].Execute(j); outputData.SetRow(j, nnSet[j].OutputData); } var s1 = new Stage2NeuralNetwork(vectors.Length, cycles, outputData, pricingData.Skip(trainLength).ToArray().Take(validationLength).ToArray().Skip(window).ToArray()); s1.Execute(); ////////////////////////////////////////////////////////////////////////// //////////////////////////////////prediction///////////////////////////// var predictedData = new DenseMatrix(vectors.Length, predictLength - window + 1); var lastPredData = new double[vectors.Length]; for (int j = 0; j < vectors.Length; j++) { double[] predictData = totalData.Row(j) .ToArray() .Skip(trainLength + validationLength) .ToArray() .Take(predictLength) .ToArray(); nnSet[j].Predict(predictData); predictedData.SetRow(j, nnSet[j].OutputData); lastPredData[j] = nnSet[j].NextPrediction; } s1.Predict(predictedData, pricingData.ToArray() .Skip(trainLength + validationLength) .ToArray() .Take(predictLength) .ToArray() .Skip(window) .ToArray()); correlation = s1.outputCorre; RMSE = s1.outputRMSE; }
/// <summary> /// Run example /// </summary> public void Run() { // Format vector output to console var formatProvider = (CultureInfo)CultureInfo.InvariantCulture.Clone(); formatProvider.TextInfo.ListSeparator = " "; // Create new empty square matrix var matrix = new DenseMatrix(10); Console.WriteLine(@"Empty matrix"); Console.WriteLine(matrix.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 1. Fill matrix by data using indexer [] var k = 0; for (var i = 0; i < matrix.RowCount; i++) { for (var j = 0; j < matrix.ColumnCount; j++) { matrix[i, j] = k++; } } Console.WriteLine(@"1. Fill matrix by data using indexer []"); Console.WriteLine(matrix.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 2. Fill matrix by data using At. The element is set without range checking. for (var i = 0; i < matrix.RowCount; i++) { for (var j = 0; j < matrix.ColumnCount; j++) { matrix.At(i, j, k--); } } Console.WriteLine(@"2. Fill matrix by data using At"); Console.WriteLine(matrix.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 3. Clone matrix var clone = matrix.Clone(); Console.WriteLine(@"3. Clone matrix"); Console.WriteLine(clone.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 4. Clear matrix clone.Clear(); Console.WriteLine(@"4. Clear matrix"); Console.WriteLine(clone.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 5. Copy matrix into another matrix matrix.CopyTo(clone); Console.WriteLine(@"5. Copy matrix into another matrix"); Console.WriteLine(clone.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 6. Get submatrix into another matrix var submatrix = matrix.SubMatrix(2, 2, 3, 3); Console.WriteLine(@"6. Copy submatrix into another matrix"); Console.WriteLine(submatrix.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 7. Get part of the row as vector. In this example: get 4 elements from row 5 starting from column 3 var row = matrix.Row(5, 3, 4); Console.WriteLine(@"7. Get part of the row as vector"); Console.WriteLine(row.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 8. Get part of the column as vector. In this example: get 3 elements from column 2 starting from row 6 var column = matrix.Column(2, 6, 3); Console.WriteLine(@"8. Get part of the column as vector"); Console.WriteLine(column.ToString("#0.00\t", formatProvider)); Console.WriteLine(); // 9. Get columns using column enumerator. If you need all columns you may use ColumnEnumerator without parameters Console.WriteLine(@"9. Get columns using column enumerator"); foreach (var keyValuePair in matrix.ColumnEnumerator(2, 4)) { Console.WriteLine(@"Column {0}: {1}", keyValuePair.Item1, keyValuePair.Item2.ToString("#0.00\t", formatProvider)); } Console.WriteLine(); // 10. Get rows using row enumerator. If you need all rows you may use RowEnumerator without parameters Console.WriteLine(@"10. Get rows using row enumerator"); foreach (var keyValuePair in matrix.RowEnumerator(4, 3)) { Console.WriteLine(@"Row {0}: {1}", keyValuePair.Item1, keyValuePair.Item2.ToString("#0.00\t", formatProvider)); } Console.WriteLine(); // 11. Convert matrix into multidimensional array var data = matrix.ToArray(); Console.WriteLine(@"11. Convert matrix into multidimensional array"); for (var i = 0; i < data.GetLongLength(0); i++) { for (var j = 0; j < data.GetLongLength(1); j++) { Console.Write(data[i, j].ToString("#0.00\t")); } Console.WriteLine(); } Console.WriteLine(); // 12. Convert matrix into row-wise array var rowwise = matrix.ToRowWiseArray(); Console.WriteLine(@"12. Convert matrix into row-wise array"); for (var i = 0; i < matrix.RowCount; i++) { for (var j = 0; j < matrix.ColumnCount; j++) { Console.Write(rowwise[(i * matrix.ColumnCount) + j].ToString("#0.00\t")); } Console.WriteLine(); } Console.WriteLine(); // 13. Convert matrix into column-wise array var columnise = matrix.ToColumnWiseArray(); Console.WriteLine(@"13. Convert matrix into column-wise array"); for (var i = 0; i < matrix.RowCount; i++) { for (var j = 0; j < matrix.ColumnCount; j++) { Console.Write(columnise[(j * matrix.RowCount) + i].ToString("#0.00\t")); } Console.WriteLine(); } Console.WriteLine(); // 14. Get matrix diagonal as vector var diagonal = matrix.Diagonal(); Console.WriteLine(@"14. Get matrix diagonal as vector"); Console.WriteLine(diagonal.ToString("#0.00\t", formatProvider)); Console.WriteLine(); }
private void ReduceCosts() { ReducedCosts = (DenseMatrix)Costs.Clone(); for (int i = 0; i < N; i++) { var min = ReducedCosts.Row(i).Minimum(); for (int j = 0; j < N; j++) { ReducedCosts[i, j] -= min; } } //todo: maybe round result for (int j = 0; j < N; j++) { var min = ReducedCosts.Column(j).Minimum(); for (int i = 0; i < N; i++) { ReducedCosts[i, j] -= min; } } //todo: maybe round result }