private void optimizeInternally(List <string> indicatorsToTry, IndicatorSelector selector)
{
while (ended == false)
{
try
{
//generator.getGeneratedIndicator(Convert.ToInt32(outcomeTimeframe / 1000 / 15), Convert.ToInt32(outcomeTimeframe * 100 / 1000));
int index = getNextIndex();
if (index >= indicatorsToTry.Count)
{
break;
}
WalkerIndicator wi = IndicatorGenerator.getIndicatorByString(indicatorsToTry[index]);
LearningIndicator li = new LearningIndicator(wi, priceData, outcomeCodeData, outcomeData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps, true);
selector.pushIndicatorStatistics(li);
}
catch (TooLittleValidDataException e)
{
//Logger.log("E:" + e.Message);
}
catch (TooLittleStatesException e)
{
//Logger.log("E:" + e.Message);
}
catch (Exception e)
{
Logger.log("FATAL:" + e.Message);
}
}
ended = true;
}
public void startRunningRandomIndicators(IndicatorGenerator generator)
{
if (running == true)
{
throw new Exception("Already running!");
}
submitResults(LearningIndicator.getPredictivePowerArrayHeader() + "usedValues;name;id");
Logger.log("Start testing indicators");
new Thread(delegate() {
running = true;
while (running)
{
//How about a genetic algo?
try {
testAndSubmitResult(generator.getGeneratedIndicator(Convert.ToInt32(outcomeTimeframe / 1000 / 15), Convert.ToInt32(outcomeTimeframe * 100 / 1000)));
}
catch (TooLittleValidDataException e)
{
Logger.log("E:" + e.Message);
}
catch (TooLittleStatesException e)
{
Logger.log("E:" + e.Message);
}
catch (Exception e)
{
Logger.log("FATAL:" + e.Message);
}
}
}).Start();
}
private void maxPpOutcomeCodeBtn_Click(object sender, EventArgs e)
{
DataminingInputDialog id = new DataminingInputDialog(new string[] { "instrument", "outcomeCodeId" }, dataminingDb.getInfo());
id.ShowDialog();
long operationSum = 0;
int operationsCount = 0;
if (id.isValidResult())
{
string instrument = id.getResult()["instrument"];
string outcomeId = id.getResult()["outcomeCodeId"];
string folderPath = Config.startupPath + "/analysis/";
if (Directory.Exists(folderPath) == false)
{
Directory.CreateDirectory(folderPath);
}
string graphFolderPath = folderPath + "ppForIndicators-" + outcomeId + "/";
Directory.CreateDirectory(graphFolderPath);
string indicatorListFilename = folderPath + "ppForIndicators-" + outcomeId + ".csv";
if (File.Exists(indicatorListFilename) == false)
{
writeTextToFile(indicatorListFilename, "OverHalf;MaxDiff;Direction;LinRegr;LogRegr;Indicator" + Environment.NewLine);
}
//Start some threads for
for (int threadId = 0; threadId < 2; threadId++) //Todo: Do 4 threads
{
new Thread(delegate()
{
while (true)
{
try
{
Stopwatch watch = new Stopwatch();
watch.Start();
WalkerIndicator indicator = IndicatorGenerator.getRandomIndicator();
string indicatorId = "mid-" + indicator.getName();
Logger.log("Start indicator: " + indicatorId, "maxPp");
setState("Max pp: avg" + Math.Round(operationSum / 1000d / (operationsCount != 0 ? operationsCount : 1)) + "s" + " n" + operationsCount);
try {
dataminingDb.addIndicator(indicator, instrument, "mid");
} catch (IndicatorNeverValidException)
{
writeTextToFile(indicatorListFilename, "x;x;x;" + indicatorId + Environment.NewLine);
Logger.log("Invalid Indicator " + indicatorId, "maxPp");
continue;
}
Logger.log("Get info", "maxPp");
DistributionRange range = dataminingDb.getInfo(indicatorId).ranges["5"];
SampleOutcomeCodeExcelGenerator excel = new SampleOutcomeCodeExcelGenerator(graphFolderPath + indicatorId + ".xls");
Logger.log("Start sampling", "maxPp");
double[] ppMethod1 = dataminingDb.getOutcomeCodeIndicatorSampling(excel, indicatorId, 20, range, outcomeId, instrument);
excel.FinishDoc();
/*double[][] inputsTraining = new double[0][], outputsTraining = new double[0][];
* dataminingDb.getInputOutputArrays(new string[] { indicatorId }, outcomeId, instrument, ref inputsTraining, ref outputsTraining, DataGroup.All, 1000 * 20, 0);
*
* double[][] inputsTest = new double[0][], outputsTest = new double[0][];
* dataminingDb.getInputOutputArrays(new string[] { indicatorId }, outcomeId, instrument, ref inputsTest, ref outputsTest, DataGroup.All, 5000, 1);
*
* double ppMethod2 = PredictivePowerAnalyzer.getPredictivePowerWithMl(inputsTraining, outputsTraining, inputsTest, outputsTest, MLMethodForPPAnalysis.LinearRegression);
*
* double ppMethod3 = PredictivePowerAnalyzer.getPredictivePowerWithMl(inputsTraining, outputsTraining, inputsTest, outputsTest, MLMethodForPPAnalysis.LogRegression);*/
Logger.log("write to file", "maxPp");
//over 0.5, maxDiff, direction
string resultLine = ppMethod1[0] + ";" + ppMethod1[1] + ";" + ppMethod1[2] + ";" + "ni" + ";" + "ni" + ";" + indicatorId;
writeTextToFile(indicatorListFilename, resultLine + Environment.NewLine);
Logger.sendImportantMessage(DateTime.Now.ToShortTimeString() + " - " + resultLine);
Logger.log("remove datasets", "maxPp");
dataminingDb.removeDataset(indicatorId, instrument);
watch.Stop();
operationSum += watch.ElapsedMilliseconds;
operationsCount++;
}
catch {
Logger.log("Error in thread method", "maxPp");
}
}
}).Start();
}
}
}
public void updateIndicators(long timeframeToLookBack, long timeframeToLookBackForIndicatorInit, IndicatorSelector indicatorSelector)
{
List <double[]> selectedPriceData = new List <double[]>();
for (int i = priceData.Count - 1; i > 0; i--)
{
if (Convert.ToInt64(priceData[i][(int)PriceDataIndeces.Date]) > timestampNow - timeframeToLookBack)
{
selectedPriceData.Insert(0, priceData[i]); //Todo: List direction correct?
}
else
{
break;
}
}
double[][] selectedPriceDataArray = selectedPriceData.ToArray();
double s;
double[][] outcomeData = OutcomeGenerator.getOutcome(selectedPriceDataArray, outcomeTimeframe, out s);
if (s < 0.6)
{
throw new Exception("s < o.6: " + s);
}
//bool[][] outcomeCodeData = OutcomeGenerator.getOutcomeCode(selectedPriceDataArray, outcomeData, outcomeCodePercent, out s);
bool[][] outcomeCodeFirstData = OutcomeGenerator.getOutcomeCodeFirst(selectedPriceDataArray, outcomeTimeframe, outcomeCodePercent, out s);
if (s < 0.6)
{
throw new Exception("s < o.6: " + s);
}
string[] indicatorIds;
//This part can be skipped by caching todo: get from outside
double hash = outcomeTimeframe + selectedPriceData[0].Sum() + selectedPriceData[selectedPriceData.Count - 1].Sum() + selectedPriceData[selectedPriceData.Count / 2].Sum();
string optimalIndicatorsFileName = cachePath + "/" + "optimalIndicatorsIn_" + hash + "_" + selectedPriceData[selectedPriceData.Count - 1][(int)PriceDataIndeces.Date] + "_" + timeframeToLookBack + "_" + outcomeCodePercent + ".txt";
if (cachePath != null && File.Exists(optimalIndicatorsFileName))
{
indicatorIds = File.ReadAllText(optimalIndicatorsFileName).Split(Environment.NewLine.ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
Logger.log("Loaded optimal indicators from file: " + optimalIndicatorsFileName);
}
else
{
//Shuffle okay indicators? todo:
Logger.log("Generated optimal indicators");
IndicatorOptimizer optimizer = new IndicatorOptimizer(selectedPriceDataArray, outcomeData, outcomeCodeFirstData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps);
indicatorIds = optimizer.getOptimizedIndicators(okayIndicators, indicatorSelector, 8);
File.WriteAllLines(optimalIndicatorsFileName, indicatorIds);
}
Logger.log("Selected indicators: ");
List <LearningIndicator> lis = new List <LearningIndicator>();
foreach (string str in indicatorIds)
{
Logger.log(str);
WalkerIndicator ind = IndicatorGenerator.getIndicatorByString(str);
if (ind.getName() != str)
{
throw new Exception(str + "!=" + ind.getName());
}
lis.Add(new LearningIndicator(ind, selectedPriceDataArray, outcomeCodeFirstData, outcomeData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps, false));
}
SignalMachine sm = new AlternativeSignalMachine(lis.ToArray()); //Todo: make accessable copy?
Logger.log("SM STATE: ##################" + Environment.NewLine + sm.getStateMessage());
//Make them up to date
List <double[]> selectedPriceDataForIndicatorInit = new List <double[]>();
for (int i = priceData.Count - 1; i > 0; i--)
{
if (Convert.ToInt64(priceData[i][(int)PriceDataIndeces.Date]) > timestampNow - timeframeToLookBackForIndicatorInit)
{
selectedPriceDataForIndicatorInit.Insert(0, priceData[i]); //Todo: List direction correct?
}
else
{
break;
}
}
foreach (double[] row in selectedPriceDataForIndicatorInit)
{
sm.pushPrice(row);
}
this.signalMachine = sm;
}
private void FindOkayIndicatorsForm_Load(object sender, EventArgs e)
{
timer1.Start();
string okayIndicatorsFile = "okayIndicators" + outcomeTimeframe + ".txt";
if (File.Exists(okayIndicatorsFile))
{
List <string> lines = File.ReadLines(okayIndicatorsFile).ToList();
foreach (string line in lines)
{
if (line != "" && line != null && line != " ")
{
found.Add(line, true);
}
}
}
DataLoader dl = new DataLoader(Config.DataPath + pair);
double[][] priceData = dl.getArray(1000 * 60 * 60 * 24 * 30l,
timeframeToTest,
minTimestep); //One month, but the second. Every 10 secs
double success;
bool[][] outcomeCodeFirstData = OutcomeGenerator.getOutcomeCodeFirst(priceData, outcomeTimeframe, outcomeCodePercent, out success);
if (success < 0.7)
{
throw new Exception("OutcomeCode low success: " + success);
}
double[][] outcomeData = OutcomeGenerator.getOutcome(priceData, outcomeTimeframe, out success);
if (success < 0.7)
{
throw new Exception("Outcome low success: " + success);
}
IndicatorGenerator generator = new IndicatorGenerator();
new Thread(delegate() {
while (true)
{
WalkerIndicator ind = generator.getGeneratedIndicator(Convert.ToInt32((outcomeTimeframe / 1000) / 10), Convert.ToInt32((outcomeTimeframe / 1000) * 100));
try
{
if (found.ContainsKey(ind.getName()) == false)
{
tried++;
new LearningIndicator(ind, priceData, outcomeCodeFirstData, outcomeData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps, false);
File.AppendAllText(okayIndicatorsFile, ind.getName() + Environment.NewLine);
found.Add(ind.getName(), true);
}
}
catch (Exception) { }
}
}).Start();
}
