private void optimizeInternally(List <string> indicatorsToTry, IndicatorSelector selector) { while (ended == false) { try { //generator.getGeneratedIndicator(Convert.ToInt32(outcomeTimeframe / 1000 / 15), Convert.ToInt32(outcomeTimeframe * 100 / 1000)); int index = getNextIndex(); if (index >= indicatorsToTry.Count) { break; } WalkerIndicator wi = IndicatorGenerator.getIndicatorByString(indicatorsToTry[index]); LearningIndicator li = new LearningIndicator(wi, priceData, outcomeCodeData, outcomeData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps, true); selector.pushIndicatorStatistics(li); } catch (TooLittleValidDataException e) { //Logger.log("E:" + e.Message); } catch (TooLittleStatesException e) { //Logger.log("E:" + e.Message); } catch (Exception e) { Logger.log("FATAL:" + e.Message); } } ended = true; }
public void startRunningRandomIndicators(IndicatorGenerator generator) { if (running == true) { throw new Exception("Already running!"); } submitResults(LearningIndicator.getPredictivePowerArrayHeader() + "usedValues;name;id"); Logger.log("Start testing indicators"); new Thread(delegate() { running = true; while (running) { //How about a genetic algo? try { testAndSubmitResult(generator.getGeneratedIndicator(Convert.ToInt32(outcomeTimeframe / 1000 / 15), Convert.ToInt32(outcomeTimeframe * 100 / 1000))); } catch (TooLittleValidDataException e) { Logger.log("E:" + e.Message); } catch (TooLittleStatesException e) { Logger.log("E:" + e.Message); } catch (Exception e) { Logger.log("FATAL:" + e.Message); } } }).Start(); }
private void maxPpOutcomeCodeBtn_Click(object sender, EventArgs e) { DataminingInputDialog id = new DataminingInputDialog(new string[] { "instrument", "outcomeCodeId" }, dataminingDb.getInfo()); id.ShowDialog(); long operationSum = 0; int operationsCount = 0; if (id.isValidResult()) { string instrument = id.getResult()["instrument"]; string outcomeId = id.getResult()["outcomeCodeId"]; string folderPath = Config.startupPath + "/analysis/"; if (Directory.Exists(folderPath) == false) { Directory.CreateDirectory(folderPath); } string graphFolderPath = folderPath + "ppForIndicators-" + outcomeId + "/"; Directory.CreateDirectory(graphFolderPath); string indicatorListFilename = folderPath + "ppForIndicators-" + outcomeId + ".csv"; if (File.Exists(indicatorListFilename) == false) { writeTextToFile(indicatorListFilename, "OverHalf;MaxDiff;Direction;LinRegr;LogRegr;Indicator" + Environment.NewLine); } //Start some threads for for (int threadId = 0; threadId < 2; threadId++) //Todo: Do 4 threads { new Thread(delegate() { while (true) { try { Stopwatch watch = new Stopwatch(); watch.Start(); WalkerIndicator indicator = IndicatorGenerator.getRandomIndicator(); string indicatorId = "mid-" + indicator.getName(); Logger.log("Start indicator: " + indicatorId, "maxPp"); setState("Max pp: avg" + Math.Round(operationSum / 1000d / (operationsCount != 0 ? operationsCount : 1)) + "s" + " n" + operationsCount); try { dataminingDb.addIndicator(indicator, instrument, "mid"); } catch (IndicatorNeverValidException) { writeTextToFile(indicatorListFilename, "x;x;x;" + indicatorId + Environment.NewLine); Logger.log("Invalid Indicator " + indicatorId, "maxPp"); continue; } Logger.log("Get info", "maxPp"); DistributionRange range = dataminingDb.getInfo(indicatorId).ranges["5"]; SampleOutcomeCodeExcelGenerator excel = new SampleOutcomeCodeExcelGenerator(graphFolderPath + indicatorId + ".xls"); Logger.log("Start sampling", "maxPp"); double[] ppMethod1 = dataminingDb.getOutcomeCodeIndicatorSampling(excel, indicatorId, 20, range, outcomeId, instrument); excel.FinishDoc(); /*double[][] inputsTraining = new double[0][], outputsTraining = new double[0][]; * dataminingDb.getInputOutputArrays(new string[] { indicatorId }, outcomeId, instrument, ref inputsTraining, ref outputsTraining, DataGroup.All, 1000 * 20, 0); * * double[][] inputsTest = new double[0][], outputsTest = new double[0][]; * dataminingDb.getInputOutputArrays(new string[] { indicatorId }, outcomeId, instrument, ref inputsTest, ref outputsTest, DataGroup.All, 5000, 1); * * double ppMethod2 = PredictivePowerAnalyzer.getPredictivePowerWithMl(inputsTraining, outputsTraining, inputsTest, outputsTest, MLMethodForPPAnalysis.LinearRegression); * * double ppMethod3 = PredictivePowerAnalyzer.getPredictivePowerWithMl(inputsTraining, outputsTraining, inputsTest, outputsTest, MLMethodForPPAnalysis.LogRegression);*/ Logger.log("write to file", "maxPp"); //over 0.5, maxDiff, direction string resultLine = ppMethod1[0] + ";" + ppMethod1[1] + ";" + ppMethod1[2] + ";" + "ni" + ";" + "ni" + ";" + indicatorId; writeTextToFile(indicatorListFilename, resultLine + Environment.NewLine); Logger.sendImportantMessage(DateTime.Now.ToShortTimeString() + " - " + resultLine); Logger.log("remove datasets", "maxPp"); dataminingDb.removeDataset(indicatorId, instrument); watch.Stop(); operationSum += watch.ElapsedMilliseconds; operationsCount++; } catch { Logger.log("Error in thread method", "maxPp"); } } }).Start(); } } }
public void updateIndicators(long timeframeToLookBack, long timeframeToLookBackForIndicatorInit, IndicatorSelector indicatorSelector) { List <double[]> selectedPriceData = new List <double[]>(); for (int i = priceData.Count - 1; i > 0; i--) { if (Convert.ToInt64(priceData[i][(int)PriceDataIndeces.Date]) > timestampNow - timeframeToLookBack) { selectedPriceData.Insert(0, priceData[i]); //Todo: List direction correct? } else { break; } } double[][] selectedPriceDataArray = selectedPriceData.ToArray(); double s; double[][] outcomeData = OutcomeGenerator.getOutcome(selectedPriceDataArray, outcomeTimeframe, out s); if (s < 0.6) { throw new Exception("s < o.6: " + s); } //bool[][] outcomeCodeData = OutcomeGenerator.getOutcomeCode(selectedPriceDataArray, outcomeData, outcomeCodePercent, out s); bool[][] outcomeCodeFirstData = OutcomeGenerator.getOutcomeCodeFirst(selectedPriceDataArray, outcomeTimeframe, outcomeCodePercent, out s); if (s < 0.6) { throw new Exception("s < o.6: " + s); } string[] indicatorIds; //This part can be skipped by caching todo: get from outside double hash = outcomeTimeframe + selectedPriceData[0].Sum() + selectedPriceData[selectedPriceData.Count - 1].Sum() + selectedPriceData[selectedPriceData.Count / 2].Sum(); string optimalIndicatorsFileName = cachePath + "/" + "optimalIndicatorsIn_" + hash + "_" + selectedPriceData[selectedPriceData.Count - 1][(int)PriceDataIndeces.Date] + "_" + timeframeToLookBack + "_" + outcomeCodePercent + ".txt"; if (cachePath != null && File.Exists(optimalIndicatorsFileName)) { indicatorIds = File.ReadAllText(optimalIndicatorsFileName).Split(Environment.NewLine.ToCharArray(), StringSplitOptions.RemoveEmptyEntries); Logger.log("Loaded optimal indicators from file: " + optimalIndicatorsFileName); } else { //Shuffle okay indicators? todo: Logger.log("Generated optimal indicators"); IndicatorOptimizer optimizer = new IndicatorOptimizer(selectedPriceDataArray, outcomeData, outcomeCodeFirstData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps); indicatorIds = optimizer.getOptimizedIndicators(okayIndicators, indicatorSelector, 8); File.WriteAllLines(optimalIndicatorsFileName, indicatorIds); } Logger.log("Selected indicators: "); List <LearningIndicator> lis = new List <LearningIndicator>(); foreach (string str in indicatorIds) { Logger.log(str); WalkerIndicator ind = IndicatorGenerator.getIndicatorByString(str); if (ind.getName() != str) { throw new Exception(str + "!=" + ind.getName()); } lis.Add(new LearningIndicator(ind, selectedPriceDataArray, outcomeCodeFirstData, outcomeData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps, false)); } SignalMachine sm = new AlternativeSignalMachine(lis.ToArray()); //Todo: make accessable copy? Logger.log("SM STATE: ##################" + Environment.NewLine + sm.getStateMessage()); //Make them up to date List <double[]> selectedPriceDataForIndicatorInit = new List <double[]>(); for (int i = priceData.Count - 1; i > 0; i--) { if (Convert.ToInt64(priceData[i][(int)PriceDataIndeces.Date]) > timestampNow - timeframeToLookBackForIndicatorInit) { selectedPriceDataForIndicatorInit.Insert(0, priceData[i]); //Todo: List direction correct? } else { break; } } foreach (double[] row in selectedPriceDataForIndicatorInit) { sm.pushPrice(row); } this.signalMachine = sm; }
private void FindOkayIndicatorsForm_Load(object sender, EventArgs e) { timer1.Start(); string okayIndicatorsFile = "okayIndicators" + outcomeTimeframe + ".txt"; if (File.Exists(okayIndicatorsFile)) { List <string> lines = File.ReadLines(okayIndicatorsFile).ToList(); foreach (string line in lines) { if (line != "" && line != null && line != " ") { found.Add(line, true); } } } DataLoader dl = new DataLoader(Config.DataPath + pair); double[][] priceData = dl.getArray(1000 * 60 * 60 * 24 * 30l, timeframeToTest, minTimestep); //One month, but the second. Every 10 secs double success; bool[][] outcomeCodeFirstData = OutcomeGenerator.getOutcomeCodeFirst(priceData, outcomeTimeframe, outcomeCodePercent, out success); if (success < 0.7) { throw new Exception("OutcomeCode low success: " + success); } double[][] outcomeData = OutcomeGenerator.getOutcome(priceData, outcomeTimeframe, out success); if (success < 0.7) { throw new Exception("Outcome low success: " + success); } IndicatorGenerator generator = new IndicatorGenerator(); new Thread(delegate() { while (true) { WalkerIndicator ind = generator.getGeneratedIndicator(Convert.ToInt32((outcomeTimeframe / 1000) / 10), Convert.ToInt32((outcomeTimeframe / 1000) * 100)); try { if (found.ContainsKey(ind.getName()) == false) { tried++; new LearningIndicator(ind, priceData, outcomeCodeFirstData, outcomeData, outcomeTimeframe, outcomeCodePercent, minPercentThreshold, learningIndicatorSteps, false); File.AppendAllText(okayIndicatorsFile, ind.getName() + Environment.NewLine); found.Add(ind.getName(), true); } } catch (Exception) { } } }).Start(); }