public DenseMatrix NormalizeData(DenseMatrix data)
{
var normalizedData = new DenseMatrix(data.RowCount, data.ColumnCount);
for (int i = 0; i < data.RowCount; i++)
{
normalizedData.SetRow(i, normalizeArrayInput[i].Process(data.Row(i).ToArray()));
}
return normalizedData;
}
private static double Error(DenseVector weights, DenseMatrix xM, DenseVector yV)
{
Debug.Assert(xM.RowCount == yV.Count);
Debug.Assert(xM.ColumnCount == weights.Count);
double wrong = 0.0;
for (int i = 0; i < xM.RowCount; i++)
{
var p = xM.Row(i) * weights;
if ((p < 0 ? -1 : 1) != yV[i])
wrong += 1;
}
return wrong / xM.RowCount;
}
public void Smooth(ref double[,] inputValues)
{
// TODO: Using the matrix works, but does a lot of data accesses. Can improve by working out all the data access myself? I might be able to cut down on number of data accesses, but not sure.
var inputMatrix = new DenseMatrix(inputValues.GetLength(0), inputValues.GetLength(1));
for (int i = 0; i < inputMatrix.RowCount; i++)
{
inputMatrix.SetRow(i, Smooth(inputMatrix.Row(i).ToArray()));
}
for (int i = 0; i < inputMatrix.ColumnCount; i++)
{
inputMatrix.SetColumn(i, Smooth(inputMatrix.Column(i).ToArray()));
}
inputValues = inputMatrix.ToArray();
}
public void FinishAndProcess()
{
var priceData = new DenseMatrix(symbols.Length, numTicks);
for (int j = 0; j < symbols.Length; j++)
{
SortedList<DateTime, Tick> d = mktData[j].data.Data;
for (int k = 0; k < d.Count; k++)
{
priceData[j, k] = d.Values[k].BidClose;
}
}
for (int i = 0; i < symbols.Length; i++)
{
results[i] = function((DenseVector) priceData.Row(i));
}
}
public static Dictionary<int, double> GetTopK2(DenseMatrix userSimilarities, int uidTarget, int K)
{
Dictionary<int, double> topK = new Dictionary<int, double>(K);
Vector<double> uidSimilarities = userSimilarities.Row(uidTarget);
double minSimilarity = double.MinValue;
int minUid = int.MinValue;
foreach (Tuple<int, double> entry in uidSimilarities.EnumerateIndexed(Zeros.AllowSkip))
{
int uid = entry.Item1;
double similarity = entry.Item2;
if (uid == uidTarget) { continue; } // A user is not a neighbor of himself
if (topK.Count < K) // Fill the top K list untill it is full
{
topK[uid] = similarity;
if (topK.Count == K)
{
// Find the least similar neighbor when it is full
minUid = topK.Aggregate((l, r) => l.Value < r.Value ? l : r).Key;
minSimilarity = topK[minUid];
}
}
else if (similarity > minSimilarity)
{
// Replace the least similar neighbor
topK.Remove(minUid); // The first time it does nothing as the minUid is not set
topK[uid] = similarity;
// Find the least similar neighbor
minUid = topK.Aggregate((l, r) => l.Value < r.Value ? l : r).Key;
minSimilarity = topK[minUid];
}
}
return topK;
}
public static void Process(FXSession session, string symbol1, string symbol2, string timeframe, int length)
{
HistoricPriceEngine h1 = new HistoricPriceEngine(session);
h1.GetLongHistoricPrices(symbol1, timeframe, length);
while (!h1.Complete)
{
Thread.Sleep(100);
}
HistoricPriceEngine h2 = new HistoricPriceEngine(session);
h2.GetLongHistoricPrices(symbol2, timeframe, length);
while (!h2.Complete)
{
Thread.Sleep(100);
}
//-----------------------
var dateTimeList = new SortedList<DateTime, int>();
Quantum q1 = h1.Data;
Quantum q2 = h2.Data;
var priceData = new DenseMatrix(2, q1.Data.Count);
for (int j = 0; j < ((q1.Data.Count <= q2.Data.Count)?q1.Data.Count:q2.Data.Count); j++ )
{
dateTimeList.Add(q1.Data.Values[j].Time, 1);
priceData[0, j] = q1.Data.Values[j].BidClose;
priceData[1, j] = q2.Data.Values[j].BidClose;
}
Vector<double> price1 = priceData.Row(0);
Vector<double> price2 = priceData.Row(1);
//Statistics.ApplyFunction((DenseVector)price1, Math.Log);
//Statistics.ApplyFunction((DenseVector)price2, Math.Log);
DenseVector norm1 = price1.ToArray().NormalizeZScore();
DenseVector norm2 = price2.ToArray().NormalizeZScore();
var newsym = new string[] {symbol1, symbol2, "spread"};
var m = new DenseMatrix(6, norm1.Count);
m.SetRow(0, norm1);
m.SetRow(1, norm2);
m.SetRow(2, (norm1 - norm2).ToArray().NormalizeZScore());
string filename = symbol1.Replace('/', '_') + "-" + symbol2.Replace('/', '_') + ".html";
Visualize.GenerateMultiPaneGraph(newsym, dateTimeList.Keys.ToArray(), m, QSConstants.DEFAULT_DATA_FILEPATH + filename,
new ChartOption[]{new ChartOption(), new ChartOption(){Layover = true, YPosition = 0}, new ChartOption(){YPosition = 1} }, null, filename + ".json");
FileUpload.UploadFileToFTP(QSConstants.DEFAULT_DATA_FILEPATH + filename, filename);
FileUpload.UploadFileToFTP(QSConstants.DEFAULT_DATA_FILEPATH + filename + ".json", filename + ".json");
double Spread = m[2, m.ColumnCount - 1];
if (Spread > 2.0 && m[2, m.ColumnCount - 2] <= 2.0)
Emailer.SendEmail(symbol1 + "-" + symbol2 + " Spread Above 2.0", "Test");
if (Spread < -2.0 && m[2, m.ColumnCount - 2] >= -2.0)
Emailer.SendEmail(symbol1 + "-" + symbol2 + " Spread Below -2.0", "Test");
}
public void Train(DenseMatrix X, DenseVector d, DenseVector Kd)
{
int R = X.RowCount;
int N = X.ColumnCount;
int U = 0; //the number of neurons in the structure
var c = new DenseMatrix(R, 1);
var sigma = new DenseMatrix(R, 1);
var Q = new DenseMatrix((R + 1), (R + 1));
var O = new DenseMatrix(1, (R + 1));
var pT_n = new DenseMatrix((R + 1), 1);
double maxPhi = 0;
int maxIndex;
var Psi = new DenseMatrix(N, 1);
Console.WriteLine("Running...");
//for each observation n in X
for (int i = 0; i < N; i++)
{
Console.WriteLine(100*(i/(double) N) + "%");
var x = new DenseVector(R);
X.Column(i, x);
//if there are neurons in structure,
//update structure recursively.
if (U == 0)
{
c = (DenseMatrix) x.ToColumnMatrix();
sigma = new DenseMatrix(R, 1, SigmaZero);
U = 1;
Psi = CalculatePsi(X, c, sigma);
UpdateStructure(X, Psi, d, ref Q, ref O);
pT_n =
(DenseMatrix)
(CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) Psi.Row(i).ToRowMatrix()))
.Transpose();
}
else
{
StructureRecurse(X, Psi, d, i, ref Q, ref O, ref pT_n);
}
bool KeepSpinning = true;
while (KeepSpinning)
{
//Calculate the error and if-part criteria
double ee = pT_n.Multiply(O)[0, 0];
double approximationError = Math.Abs(d[i] - ee);
DenseVector Phi;
double SumPhi;
CalculatePhi(x, c, sigma, out Phi, out SumPhi);
maxPhi = Phi.Maximum();
maxIndex = Phi.MaximumIndex();
if (approximationError > delta)
{
if (maxPhi < threshold)
{
var tempSigma = new DenseVector(R);
sigma.Column(maxIndex, tempSigma);
double minSigma = tempSigma.Minimum();
int minIndex = tempSigma.MinimumIndex();
sigma[minIndex, maxIndex] = k_sigma*minSigma;
Psi = CalculatePsi(X, c, sigma);
UpdateStructure(X, Psi, d, ref Q, ref O);
var psi = new DenseVector(Psi.ColumnCount);
Psi.Row(i, psi);
pT_n =
(DenseMatrix)
CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix())
.Transpose();
}
else
{
//add a new neuron and update strucutre
double distance = 0;
var cTemp = new DenseVector(R);
var sigmaTemp = new DenseVector(R);
//foreach input variable
for (int j = 0; j < R; j++)
{
distance = Math.Abs(x[j] - c[j, 0]);
int distanceIndex = 0;
//foreach neuron past 1
for (int k = 1; k < U; k++)
{
if ((Math.Abs(x[j] - c[j, k])) < distance)
{
distanceIndex = k;
distance = Math.Abs(x[j] - c[j, k]);
}
}
if (distance < Kd[j])
{
cTemp[j] = c[j, distanceIndex];
sigmaTemp[j] = sigma[j, distanceIndex];
}
else
{
cTemp[j] = x[j];
sigmaTemp[j] = distance;
}
}
//end foreach
c = (DenseMatrix) c.InsertColumn(c.ColumnCount - 1, cTemp);
sigma = (DenseMatrix) sigma.InsertColumn(sigma.ColumnCount - 1, sigmaTemp);
Psi = CalculatePsi(X, c, sigma);
UpdateStructure(X, Psi, d, ref Q, ref O);
U++;
KeepSpinning = false;
}
}
else
{
if (maxPhi < threshold)
{
var tempSigma = new DenseVector(R);
sigma.Column(maxIndex, tempSigma);
double minSigma = tempSigma.Minimum();
int minIndex = tempSigma.MinimumIndex();
sigma[minIndex, maxIndex] = k_sigma*minSigma;
Psi = CalculatePsi(X, c, sigma);
UpdateStructure(X, Psi, d, ref Q, ref O);
var psi = new DenseVector(Psi.ColumnCount);
Psi.Row(i, psi);
pT_n =
(DenseMatrix)
CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix())
.Transpose();
}
else
{
KeepSpinning = false;
}
}
}
}
out_C = c;
out_O = O;
out_Sigma = sigma;
Console.WriteLine("Done.");
}
public void StructureRecurse(DenseMatrix X, DenseMatrix Psi, DenseVector d, int n, ref DenseMatrix Q,
ref DenseMatrix O, ref DenseMatrix pT_n)
{
//O = O(t-1) O_enxt = O(t)
//o should be a column vector ( in matrix form)
var x = new DenseVector(X.RowCount);
var psi = new DenseVector(Psi.ColumnCount);
X.Column(n, x);
Psi.Row(n, psi);
DenseMatrix p_n = CalculateGreatPsi((DenseMatrix) x.ToColumnMatrix(), (DenseMatrix) psi.ToRowMatrix());
pT_n = (DenseMatrix) p_n.Transpose();
double ee = Math.Abs(d[n] - (pT_n.Multiply(O))[0, 0]);
double temp = 1 + (pT_n.Multiply(Q)).Multiply(p_n)[0, 0];
double ae = Math.Abs(ee/temp);
if (ee >= ae)
{
var L = (DenseMatrix) Q.Multiply(p_n).Multiply(1/temp);
Q = (DenseMatrix) ((DenseMatrix.Identity(Q.RowCount).Subtract(L.Multiply(pT_n))).Multiply(Q));
O = (DenseMatrix) O.Add(L*ee);
}
else
{
Q = (DenseMatrix) DenseMatrix.Identity(Q.RowCount).Multiply(Q);
}
}
public void Predict(DenseMatrix newPredictData, double[] newPredictPrices)
{
double error = 0;
int c = 0;
var newNormalizedPredictData = new DenseMatrix(newPredictData.RowCount, newPredictData.ColumnCount,
double.NaN);
for (int i = 0; i < newPredictData.RowCount; i++)
{
newNormalizedPredictData.SetRow(i, normalizeArrayInput[i].Process(newPredictData.Row(i).ToArray()));
}
double[] normalizedPrices = normalizeArrayOutput.Process(newPredictPrices);
var d = new DenseMatrix(2, normalizedPrices.Length + 1, double.NaN);
int count = 0;
for (int i = 0; i < normalizedPrices.Length; i++)
{
// calculate based on actual data
IMLData input = new BasicMLData(inputs);
for (int j = 0; j < input.Count; j++)
{
input.Data[j] = newNormalizedPredictData[j, i];
}
IMLData output = network.Compute(input);
double prediction = output.Data[0];
error +=
Math.Pow(
(normalizeArrayOutput.Stats.DeNormalize(prediction) - newPredictPrices[i])/newPredictPrices[i],
2);
c++;
d[0, count] = newPredictPrices[i];
d[1, count] = normalizeArrayOutput.Stats.DeNormalize(prediction);
count++;
}
/////////////////////////////////////////////////////////////////
IMLData input1 = new BasicMLData(inputs);
for (int j = 0; j < input1.Count; j++)
{
input1.Data[j] = newNormalizedPredictData[j, newNormalizedPredictData.ColumnCount - 1];
}
IMLData output1 = network.Compute(input1);
d[1, count] = normalizeArrayOutput.Stats.DeNormalize(output1.Data[0]);
/////////////////////////////////////////////////////////////////
error /= c;
error = Math.Pow(error, .5);
Console.WriteLine(error);
string[] symbols = {"actual", "predicted"};
Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0),
"C:\\Sangar\\resultfinal.html");
outputCorre =
StatisticsExtension.Correlation(d.Row(0).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn(),
d.Row(1).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn());
Console.WriteLine("ST2 Correlation: " + outputCorre);
outputRMSE = error;
Console.WriteLine("Predicted return for D+1:" +
(d[1, d.ColumnCount - 1] - d[1, d.ColumnCount - 2])/d[1, d.ColumnCount - 2]*100 +
" percent");
}
public Krige(DenseMatrix xx, DenseMatrix yy, Powvargram vg)
{
// Initializer
x = xx;
vgram = vg;
n = xx.RowCount;
d = xx.ColumnCount;
dstar = new DenseMatrix(n + 1, 1);
vstar = new DenseMatrix(n + 1, 1);
v = new DenseMatrix(n + 1, n + 1);
y = new DenseMatrix(n + 1, n);
// Building matrix V,Y
for (int i = 0; i < n; i++)
{
y[i, 0] = yy[i, 0];
for (int j = i; j < n; j++)
{
double vvv = vgram.GetV((x.Row(i) - x.Row(j)).Norm(2));
v[i, j] = vvv;
v[j, i] = vvv;
}
v[i, n] = 1.0;
v[n, i] = 1.0;
}
v[n, n] = 0.0;
y[n, 0] = 0.0;
vi = v.Inverse();
yvi = vi.Transpose() * y;
}
static Tuple<double, double> RunPLAvsSVM(int experiments, int points)
{
const int TEST_POINTS = 10000;
Random rnd = new Random();
long svmWins = 0, svCount = 0;
for (int i = 1; i <= experiments; i++)
{
//pick a random line y = a * x + b
double x1 = rnd.NextDouble(), y1 = rnd.NextDouble(), x2 = rnd.NextDouble(), y2 = rnd.NextDouble();
var Wf = new DenseVector(3);
Wf[0] = 1;
Wf[1] = (y1 - y2) / (x1 * y2 - y1 * x2);
Wf[2] = (x2 - x1) / (x1 * y2 - y1 * x2);
Func<MathNet.Numerics.LinearAlgebra.Generic.Vector<double>, int> f = x => Wf.DotProduct(x) >= 0 ? 1 : -1;
//generate training set of N random points
var X = new DenseMatrix(points, 3);
do
for (int j = 0; j < points; j++)
{
X[j, 0] = 1;
X[j, 1] = rnd.NextDouble() * 2 - 1;
X[j, 2] = rnd.NextDouble() * 2 - 1;
}
while (Enumerable.Range(0, X.RowCount).All(j => f(X.Row(0)) == f(X.Row(j))));
var W = new DenseVector(3);
Func<MathNet.Numerics.LinearAlgebra.Generic.Vector<double>, int> h = x => W.DotProduct(x) >= 0 ? 1 : -1;
//run Perceptron
int k = 1;
while (Enumerable.Range(0, points).Any(j => h(X.Row(j)) != f(X.Row(j))))
{
//find all misclasified points
int[] M = Enumerable.Range(0, points).Where(j => h(X.Row(j)) != f(X.Row(j))).ToArray();
int m = M[rnd.Next(0, M.Length)];
int sign = f(X.Row(m));
W[0] += sign;
W[1] += sign * X[m, 1];
W[2] += sign * X[m, 2];
k++;
}
//calculate P[f(Xtest) != h(Xtest)]
DenseVector Xtest = new DenseVector(3);
Xtest[0] = 1;
int matches = 0;
for (int j = 0; j < TEST_POINTS; j++)
{
Xtest[1] = rnd.NextDouble() * 2 - 1;
Xtest[2] = rnd.NextDouble() * 2 - 1;
if (f(Xtest) == h(Xtest)) matches++;
}
double Ppla = (matches + 0.0) / TEST_POINTS;
//Run SVM
var prob = new svm_problem() {
x = Enumerable.Range(0, points).Select(j =>
new svm_node[] {
new svm_node() { index = 0, value = X[j, 1] },
new svm_node() { index = 1, value = X[j, 2] } }).ToArray(),
y = Enumerable.Range(0, points).Select(j => (double)f(X.Row(j))).ToArray(),
l = points };
var model = svm.svm_train(prob, new svm_parameter()
{
svm_type = (int)SvmType.C_SVC,
kernel_type = (int)KernelType.LINEAR,
C = 1000000,
eps = 0.001,
shrinking = 0
});
//calculate P[f(Xtest) != h_svm(Xtest)]
svm_node[] Xsvm = new svm_node[] {
new svm_node() { index = 0, value = 1.0 },
new svm_node() { index = 1, value = 1.0 } };
matches = 0;
for (int j = 0; j < TEST_POINTS; j++)
{
Xtest[1] = rnd.NextDouble() * 2 - 1;
Xsvm[0].value = Xtest[1];
Xtest[2] = rnd.NextDouble() * 2 - 1;
Xsvm[1].value = Xtest[2];
if (f(Xtest) == (svm.svm_predict(model, Xsvm) > 0 ? 1 : -1)) matches++;
}
double Psvm = (matches + 0.0) / TEST_POINTS;
svCount += model.l;
if (Psvm >= Ppla) svmWins++;
}
return Tuple.Create((svmWins + 0.0) / experiments, (svCount + 0.0) / experiments);
}
public void Predict(double[] newPredictData)
{
double error = 0;
int c = 0;
double[] newNormalizedData = normalizeArray.Process(newPredictData);
var d = new DenseMatrix(2, newNormalizedData.Length - WindowSize + 1, double.NaN);
int count = 0;
for (int i = WindowSize; i < newNormalizedData.Length; i++)
{
// calculate based on actual data
IMLData input = new BasicMLData(WindowSize);
for (int j = 0; j < input.Count; j++)
{
input.Data[j] = newNormalizedData[(i - WindowSize) + j];
}
IMLData output = network.Compute(input);
double prediction = output.Data[0];
error += Math.Pow((normalizeArray.Stats.DeNormalize(prediction) - newPredictData[i])/newPredictData[i],
2);
c++;
d[0, count] = newPredictData[i];
d[1, count] = normalizeArray.Stats.DeNormalize(prediction);
count++;
}
///////////////////////////////////////////////////////////////////////////////
var lastData = new double[WindowSize];
int count1 = 0;
for (int i = newNormalizedData.Length - WindowSize; i < newNormalizedData.Length; i++)
{
lastData[count1++] = newNormalizedData[i];
}
IMLData input1 = new BasicMLData(WindowSize);
for (int j = 0; j < input1.Count; j++)
{
input1.Data[j] = lastData[j];
}
IMLData output1 = network.Compute(input1);
d[1, count] = normalizeArray.Stats.DeNormalize(output1.Data[0]);
/////////////////////////////////////////////////////////////////////////////////
error /= c;
error = Math.Pow(error, .5);
Console.WriteLine(error);
OutputData = d.Row(1).ToArray();
string[] symbols = {"actual", "predicted"};
Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0),
QSConstants.DEFAULT_DATA_FILEPATH + write + ".html");
Console.WriteLine("ST1 Correlation: " +
StatisticsExtension.Correlation(
d.Row(0).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn(),
d.Row(1).ToArray().Take(d.ColumnCount - 1).ToArray().RawRateOfReturn()));
}
public void Predict()
{
double error = 0;
int c = 0;
var d = new DenseMatrix(2, _normalizedPredictionData.Length - WindowSize);
int count = 0;
for (int i = WindowSize; i < _normalizedPredictionData.Length; i++)
{
// calculate based on actual data
IMLData input = new BasicMLData(WindowSize);
for (int j = 0; j < input.Count; j++)
{
input.Data[j] = _normalizedPredictionData[(i - WindowSize) + j];
}
IMLData output = network.Compute(input);
double prediction = output.Data[0];
error += Math.Pow((normalizeArray.Stats.DeNormalize(prediction) - predictionData[i])/predictionData[i],
2);
c++;
d[0, count] = predictionData[i];
d[1, count] = normalizeArray.Stats.DeNormalize(prediction);
count++;
}
error /= c;
error = Math.Pow(error, .5);
Console.WriteLine(error);
OutputData = d.Row(1).ToArray();
string[] symbols = {"actual", "predicted"};
Visualize.GeneratePredictionGraph(symbols, d, new DateTime(), new TimeSpan(24, 0, 0),
QSConstants.DEFAULT_DATA_FILEPATH + write + ".html");
}
public void FinishAndProcess()
{
try
{
var priceData = new DenseMatrix(symbols.Length, numTicks);
for (int j = 0; j < symbols.Length; j++)
{
SortedList<DateTime, Tick> d = mktData[j].data.Data;
for (int k = 0; k < d.Count; k++)
{
//if (!symbols[j].Substring(0, 3).Equals("USD")) priceData[j, k] = 1/d.Values[k].BidClose;
priceData[j, k] = d.Values[k].BidOpen;
}
}
Vector<double> price1 = priceData.Row(0);
Vector<double> price2 = priceData.Row(1);
//Statistics.ApplyFunction((DenseVector)price1, Math.Log);
//Statistics.ApplyFunction((DenseVector)price2, Math.Log);
DenseVector norm1 = price1.ToArray().NormalizeZScore();
DenseVector norm2 = price2.ToArray().NormalizeZScore();
var newsym = new string[symbols.Length + 4];
for (int i = 0; i < symbols.Length; i++) newsym[i] = symbols[i];
newsym[2] = "spread";
newsym[3] = "EMA5";
newsym[4] = "EMA15";
newsym[5] = "EMA30";
var m = new DenseMatrix(6, norm1.Count);
m.SetRow(0, norm1);
m.SetRow(1, norm2);
m.SetRow(2, (norm1 - norm2).ToArray().NormalizeZScore());
m.SetRow(3, EMA.CalcEMA(m.Row(2).ToArray(), 5));
m.SetRow(4, EMA.CalcEMA(m.Row(2).ToArray(), 15));
m.SetRow(5, EMA.CalcEMA(m.Row(2).ToArray(), 30));
string filename = symbols[0].Replace('/', '_') + "-" + symbols[1].Replace('/', '_') + ".html";
((DenseVector) m.Row(0)).GenerateSimpleGraph("C:\\Sangar\\result.html");
Visualize.GenerateMultiSymbolGraph(newsym, m, DateTime.Now.AddSeconds(-60*5*300), new TimeSpan(0, 5, 0),
"C:\\Sangar\\" + filename);
FileUpload.UploadFileToFTP("C:\\Sangar\\" + filename, filename);
Spread = m[2, m.ColumnCount - 1];
if (Spread > 2.0 && m[2, m.ColumnCount - 2] <= 2.0)
Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Above 2.0", "Test");
if (Spread < -2.0 && m[2, m.ColumnCount - 2] >= -2.0)
Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Below -2.0", "Test");
//if (m[2, m.ColumnCount - 1] < 0.5 && m[2, m.ColumnCount - 2] >= 0.5)
// Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Below 0.5", "Test");
//if (m[2, m.ColumnCount - 1] > -0.5 && m[2, m.ColumnCount - 2] <= -0.5)
// Emailer.SendEmail(symbols[0] + "-" + symbols[1] + " Spread Above -0.5", "Test");
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
public void FinishAndProcess()
{
try
{
var priceData = new DenseMatrix(symbols.Length, numTicks);
for (int j = 0; j < symbols.Length; j++)
{
SortedList<DateTime, Tick> d = mktData[j].data.Data;
for (int k = 0; k < d.Count; k++)
{
priceData[j, k] = d.Values[k].BidClose;
}
}
for (int i = 0; i < symbols.Length; i++)
{
for (int j = 0; j < symbols.Length; j++)
{
double[] pDatai = priceData.Row(i).ToArray();
double[] pDataj = priceData.Row(j).ToArray();
switch (cType)
{
case CovarianceType.LogReturn:
{
pDatai = priceData.Row(i).ToArray().LogRateOfReturn();
pDataj = priceData.Row(j).ToArray().LogRateOfReturn();
break;
}
case CovarianceType.RawReturn:
{
pDatai = priceData.Row(i).ToArray().RawRateOfReturn();
pDataj = priceData.Row(j).ToArray().RawRateOfReturn();
break;
}
}
correlation[i, j] = StatisticsExtension.Correlation(pDatai, pDataj);
covariance[i, j] = StatisticsExtension.Covariance((DenseVector) priceData.Row(i),
(DenseVector) priceData.Row(j));
}
}
Visualize.GenerateHeatMatrix(symbols, correlation, "C:\\Users\\Ethan\\Work\\QuantSysdata.html");
Console.WriteLine("Finished Generating Correlation Matrix.");
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
/// <summary>
/// Generate a random n-class classification problem.
/// </summary>
/// <param name="nSamples">The number of samples.</param>
/// <param name="nFeatures">The total number of features. These comprise <paramref name="nInformative"/>
/// informative features, <paramref name="nRedundant"/> redundant features, <paramref name="nRepeated"/>
/// dupplicated features and `<paramref name="nFeatures"/>-<paramref name="nInformative"/>-<paramref name="nRedundant"/>-
/// <paramref name="nRepeated"/>` useless features drawn at random.</param>
/// <param name="nInformative">The number of informative features. Each class is composed of a number
/// of gaussian clusters each located around the vertices of a hypercube
/// in a subspace of dimension <paramref name="nInformative"/>. For each cluster,
/// informative features are drawn independently from N(0, 1) and then
/// randomly linearly combined in order to add covariance. The clusters
/// are then placed on the vertices of the hypercube.</param>
/// <param name="nRedundant">The number of redundant features. These features are generated as
/// random linear combinations of the informative features.</param>
/// <param name="nRepeated"> The number of dupplicated features, drawn randomly from the informative
/// and the redundant features.
/// </param>
/// <param name="nClasses">The number of classes (or labels) of the classification problem.</param>
/// <param name="nClustersPerClass">The number of clusters per class.</param>
/// <param name="weights">The proportions of samples assigned to each class. If None, then
/// classes are balanced. Note that if `len(weights) == n_classes - 1`,
/// then the last class weight is automatically inferred.
/// </param>
/// <param name="flipY">The fraction of samples whose class are randomly exchanged.</param>
/// <param name="classSep">The factor multiplying the hypercube dimension.</param>
/// <param name="hypercube">If True, the clusters are put on the vertices of a hypercube. If
/// False, the clusters are put on the vertices of a random polytope.</param>
/// <param name="shift">Shift all features by the specified value. If None, then features
/// are shifted by a random value drawn in [-class_sep, class_sep].</param>
/// <param name="scale">Multiply all features by the specified value. If None, then features
/// are scaled by a random value drawn in [1, 100]. Note that scaling
/// happens after shifting.
/// </param>
/// <param name="shuffle">Shuffle the samples and the features.</param>
/// <param name="randomState">Random generator.</param>
/// <returns>array of shape [n_samples]
/// The integer labels for class membership of each sample.</returns>
/// <remarks>
/// The algorithm is adapted from Guyon [1] and was designed to generate
/// the "Madelon" dataset.
/// References
/// ----------
/// .. [1] I. Guyon, "Design of experiments for the NIPS 2003 variable
/// selection benchmark", 2003.
/// </remarks>
public static Classification MakeClassification(
int nSamples = 100,
int nFeatures = 20,
int nInformative = 2,
int nRedundant = 2,
int nRepeated = 0,
int nClasses = 2,
int nClustersPerClass = 2,
List<double> weights = null,
double flipY = 0.01,
double classSep = 1.0,
bool hypercube = true,
double? shift = 0.0,
double? scale = 1.0,
bool shuffle = true,
Random randomState = null)
{
var generator = randomState ?? new Random();
// Count features, clusters and samples
if (nInformative + nRedundant + nRepeated > nFeatures)
{
throw new ArgumentException("Number of informative, redundant and repeated " +
"features must sum to less than the number of total" +
" features");
}
if (nInformative * nInformative < nClasses * nClustersPerClass)
{
throw new ArgumentException(
"n_classes * n_clusters_per_class must" +
"be smaller or equal 2 ** n_informative");
}
if (weights != null && !new[] { nClasses, nClasses - 1 }.Contains(weights.Count))
{
throw new ArgumentException("Weights specified but incompatible with number of classes.");
}
int nUseless = nFeatures - nInformative - nRedundant - nRepeated;
int nClusters = nClasses * nClustersPerClass;
if (weights != null && weights.Count == nClasses - 1)
{
weights.Add(1.0 - weights.Sum());
}
if (weights == null)
{
weights = Enumerable.Repeat(1.0 / nClasses, nClasses).ToList();
weights[weights.Count - 1] = 1.0 - weights.Take(weights.Count - 1).Sum();
}
var nSamplesPerCluster = new List<int>();
for (int k = 0; k < nClusters; k++)
{
nSamplesPerCluster.Add(
(int)(nSamples * weights[k % nClasses] / nClustersPerClass));
}
for (int i = 0; i < nSamples - nSamplesPerCluster.Sum(); i++)
{
nSamplesPerCluster[i % nClusters] += 1;
}
// Intialize X and y
Matrix x = new DenseMatrix(nSamples, nFeatures);
int[] y = new int[nSamples];
// Build the polytope
Matrix c = new DenseMatrix(1 << nInformative, nInformative);
for (int i = 0; i < 1 << nInformative; i++)
{
var row = new DenseVector(nInformative);
for (int bitN = 0; bitN < nInformative; bitN++)
{
row[bitN] = (i & (1 << bitN)) == 1 ? classSep : -classSep;
}
c.SetRow(i, row);
}
if (!hypercube)
{
for (int k = 0; k < nClusters; k++)
{
c.SetRow(k, c.Row(k) * generator.NextDouble());
}
for (int f = 0; f < nInformative; f++)
{
c.SetColumn(f, c.Column(f) * generator.NextDouble());
}
}
// todo:
// generator.shuffle(C)
// Loop over all clusters
int pos = 0;
int posEnd = 0;
for (int k = 0; k < nClusters; k++)
{
// Number of samples in cluster k
int nSamplesK = nSamplesPerCluster[k];
// Define the range of samples
pos = posEnd;
posEnd = pos + nSamplesK;
// Assign labels
for (int l = pos; l < posEnd; l++)
{
y[l] = k % nClasses;
}
// Draw features at random
var subMatrix = DenseMatrix.CreateRandom(
nSamplesK,
nInformative,
new Normal { RandomSource = generator });
x.SetSubMatrix(pos, nSamplesK, 0, nInformative, subMatrix);
// Multiply by a random matrix to create co-variance of the features
var uniform = new ContinuousUniform(-1, 1) { RandomSource = generator };
Matrix a = DenseMatrix.CreateRandom(nInformative, nInformative, uniform);
x.SetSubMatrix(
pos,
nSamplesK,
0,
nInformative,
x.SubMatrix(pos, nSamplesK, 0, nInformative) * a);
// Shift the cluster to a vertice
var v = x.SubMatrix(pos, nSamplesK, 0, nInformative).AddRowVector(c.Row(k));
x.SetSubMatrix(pos, nSamplesK, 0, nInformative, v);
}
// Create redundant features
if (nRedundant > 0)
{
var uniform = new ContinuousUniform(-1, 1) { RandomSource = generator };
Matrix b = DenseMatrix.CreateRandom(nInformative, nRedundant, uniform);
x.SetSubMatrix(
0,
x.RowCount,
nInformative,
nRedundant,
x.SubMatrix(0, x.RowCount, 0, nInformative) * b);
}
// Repeat some features
if (nRepeated > 0)
{
int n = nInformative + nRedundant;
for (int i = 0; i < nRepeated; i++)
{
int r = (int)((generator.Next(nRepeated) * (n - 1)) + 0.5);
x.SetColumn(i, x.Column(r));
}
}
// Fill useless features
var denseMatrix = DenseMatrix.CreateRandom(nSamples, nUseless, new Normal { RandomSource = generator });
x.SetSubMatrix(0, nSamples, nFeatures - nUseless, nUseless, denseMatrix);
// Randomly flip labels
if (flipY >= 0.0)
{
for (int i = 0; i < nSamples; i++)
{
if (generator.NextDouble() < flipY)
{
y[i] = generator.Next(nClasses);
}
}
}
// Randomly shift and scale
bool constantShift = shift != null;
bool constantScale = scale != null;
for (int f = 0; f < nFeatures; f++)
{
if (!constantShift)
{
shift = ((2 * generator.NextDouble()) - 1) * classSep;
}
if (!constantScale)
{
scale = 1 + (100 * generator.NextDouble());
}
x.SetColumn(f, (x.Column(f) + shift.Value) * scale.Value);
}
// Randomly permute samples and features
// todo:
/*
if (shuffle)
{
X, y = util_shuffle(X, y, random_state=generator)
indices = np.arange(n_features)
generator.shuffle(indices)
X[:, :] = X[:, indices]
}*/
return new Classification { X = x, Y = y };
}
public void Execute()
{
var nnSet = new Stage1NeuralNetwork[vectors.Length];
int trainLength = 2000;
int validationLength = 500;
int predictLength = 500;
int useLength = 3000;
var totalData = new DenseMatrix(vectors.Length, useLength);
var outputData = new DenseMatrix(vectors.Length, validationLength - window);
/////////////////populate the actual price data we want to predict
var pricingData = new double[useLength];
for (int i = 2; i < 2 + useLength; i++)
{
pricingData[i - 2] = (double) data[useLength + 3 - i, 5];
}
double[] returnpricingData = pricingData.RawRateOfReturn();
for (int i = 0; i < returnpricingData.Length; i++) pricingData[i + 1] = returnpricingData[i];
pricingData[0] = 0;
////////////////////////training and validation////////////////////////
for (int i = 2; i < 2 + useLength; i++)
{
for (int j = 0; j < vectors.Length; j++)
{
totalData[j, i - 2] = (double) data[useLength + 3 - i, vectors[j]];
}
}
for (int j = 0; j < vectors.Length; j++)
{
double[] train = totalData.Row(j).ToArray().Take(trainLength).ToArray();
double[] validate =
totalData.Row(j).ToArray().Skip(trainLength).ToArray().Take(validationLength).ToArray();
nnSet[j] = new Stage1NeuralNetwork(window, cycles, train, validate);
nnSet[j].Execute(j);
outputData.SetRow(j, nnSet[j].OutputData);
}
var s1 = new Stage2NeuralNetwork(vectors.Length, cycles, outputData,
pricingData.Skip(trainLength).ToArray().Take(validationLength).ToArray().Skip(window).ToArray());
s1.Execute();
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////prediction/////////////////////////////
var predictedData = new DenseMatrix(vectors.Length, predictLength - window + 1);
var lastPredData = new double[vectors.Length];
for (int j = 0; j < vectors.Length; j++)
{
double[] predictData =
totalData.Row(j)
.ToArray()
.Skip(trainLength + validationLength)
.ToArray()
.Take(predictLength)
.ToArray();
nnSet[j].Predict(predictData);
predictedData.SetRow(j, nnSet[j].OutputData);
lastPredData[j] = nnSet[j].NextPrediction;
}
s1.Predict(predictedData,
pricingData.ToArray()
.Skip(trainLength + validationLength)
.ToArray()
.Take(predictLength)
.ToArray()
.Skip(window)
.ToArray());
correlation = s1.outputCorre;
RMSE = s1.outputRMSE;
}
/// <summary>
/// Run example
/// </summary>
public void Run()
{
// Format vector output to console
var formatProvider = (CultureInfo)CultureInfo.InvariantCulture.Clone();
formatProvider.TextInfo.ListSeparator = " ";
// Create new empty square matrix
var matrix = new DenseMatrix(10);
Console.WriteLine(@"Empty matrix");
Console.WriteLine(matrix.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 1. Fill matrix by data using indexer []
var k = 0;
for (var i = 0; i < matrix.RowCount; i++)
{
for (var j = 0; j < matrix.ColumnCount; j++)
{
matrix[i, j] = k++;
}
}
Console.WriteLine(@"1. Fill matrix by data using indexer []");
Console.WriteLine(matrix.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 2. Fill matrix by data using At. The element is set without range checking.
for (var i = 0; i < matrix.RowCount; i++)
{
for (var j = 0; j < matrix.ColumnCount; j++)
{
matrix.At(i, j, k--);
}
}
Console.WriteLine(@"2. Fill matrix by data using At");
Console.WriteLine(matrix.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 3. Clone matrix
var clone = matrix.Clone();
Console.WriteLine(@"3. Clone matrix");
Console.WriteLine(clone.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 4. Clear matrix
clone.Clear();
Console.WriteLine(@"4. Clear matrix");
Console.WriteLine(clone.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 5. Copy matrix into another matrix
matrix.CopyTo(clone);
Console.WriteLine(@"5. Copy matrix into another matrix");
Console.WriteLine(clone.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 6. Get submatrix into another matrix
var submatrix = matrix.SubMatrix(2, 2, 3, 3);
Console.WriteLine(@"6. Copy submatrix into another matrix");
Console.WriteLine(submatrix.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 7. Get part of the row as vector. In this example: get 4 elements from row 5 starting from column 3
var row = matrix.Row(5, 3, 4);
Console.WriteLine(@"7. Get part of the row as vector");
Console.WriteLine(row.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 8. Get part of the column as vector. In this example: get 3 elements from column 2 starting from row 6
var column = matrix.Column(2, 6, 3);
Console.WriteLine(@"8. Get part of the column as vector");
Console.WriteLine(column.ToString("#0.00\t", formatProvider));
Console.WriteLine();
// 9. Get columns using column enumerator. If you need all columns you may use ColumnEnumerator without parameters
Console.WriteLine(@"9. Get columns using column enumerator");
foreach (var keyValuePair in matrix.ColumnEnumerator(2, 4))
{
Console.WriteLine(@"Column {0}: {1}", keyValuePair.Item1, keyValuePair.Item2.ToString("#0.00\t", formatProvider));
}
Console.WriteLine();
// 10. Get rows using row enumerator. If you need all rows you may use RowEnumerator without parameters
Console.WriteLine(@"10. Get rows using row enumerator");
foreach (var keyValuePair in matrix.RowEnumerator(4, 3))
{
Console.WriteLine(@"Row {0}: {1}", keyValuePair.Item1, keyValuePair.Item2.ToString("#0.00\t", formatProvider));
}
Console.WriteLine();
// 11. Convert matrix into multidimensional array
var data = matrix.ToArray();
Console.WriteLine(@"11. Convert matrix into multidimensional array");
for (var i = 0; i < data.GetLongLength(0); i++)
{
for (var j = 0; j < data.GetLongLength(1); j++)
{
Console.Write(data[i, j].ToString("#0.00\t"));
}
Console.WriteLine();
}
Console.WriteLine();
// 12. Convert matrix into row-wise array
var rowwise = matrix.ToRowWiseArray();
Console.WriteLine(@"12. Convert matrix into row-wise array");
for (var i = 0; i < matrix.RowCount; i++)
{
for (var j = 0; j < matrix.ColumnCount; j++)
{
Console.Write(rowwise[(i * matrix.ColumnCount) + j].ToString("#0.00\t"));
}
Console.WriteLine();
}
Console.WriteLine();
// 13. Convert matrix into column-wise array
var columnise = matrix.ToColumnWiseArray();
Console.WriteLine(@"13. Convert matrix into column-wise array");
for (var i = 0; i < matrix.RowCount; i++)
{
for (var j = 0; j < matrix.ColumnCount; j++)
{
Console.Write(columnise[(j * matrix.RowCount) + i].ToString("#0.00\t"));
}
Console.WriteLine();
}
Console.WriteLine();
// 14. Get matrix diagonal as vector
var diagonal = matrix.Diagonal();
Console.WriteLine(@"14. Get matrix diagonal as vector");
Console.WriteLine(diagonal.ToString("#0.00\t", formatProvider));
Console.WriteLine();
}
private void ReduceCosts()
{
ReducedCosts = (DenseMatrix)Costs.Clone();
for (int i = 0; i < N; i++)
{
var min = ReducedCosts.Row(i).Minimum();
for (int j = 0; j < N; j++)
{
ReducedCosts[i, j] -= min;
}
}
//todo: maybe round result
for (int j = 0; j < N; j++)
{
var min = ReducedCosts.Column(j).Minimum();
for (int i = 0; i < N; i++)
{
ReducedCosts[i, j] -= min;
}
}
//todo: maybe round result
}