/*
*
+------+. +------+ +------+ +------+ .+------+
|`. | `. |\ |\ | | /| /| .' | .'|
| `+--+---+ | +----+-+ +------+ +-+----+ | +---+--+' |
| | | | | | | | | | | | | | | | | |
+---+--+. | +-+----+ | +------+ | +----+-+ | .+--+---+
| `. | `.| \| \| | | |/ |/ |.' | .'
| `+------+ +------+ +------+ +------+ +------+'
*/
public void GenerateTrainData()
{
if (!Data.TALibMethods.Any())
{
return;
}
do
{
again:
Program.Form.ConfigurationClear();
if (Program.Form.debugView.Visible)
{
Program.Form.debugView.Invoke((MethodInvoker)(() =>
{
noDelayEnabled = Program.Form.nodelayCheckbox.Checked;
}));
}
threadSleepTime = 0; // GetIdleTickCount() >= Configuration.SleepCheckTime ? 0 : Configuration.SleepTime;
Program.Form.SetStatus($"generating functions list, sleepTime={threadSleepTime}");
class1 = class2 = class0 = 0;
Data.FunctionConfiguration.Clear();
Program.Form.debugView.Invoke((MethodInvoker)(() => { Program.Form.debugView.Items.Clear(); }));
SetupFunctions(randomSeed);
selectedInputDimension = XRandom.next(8, XRandom.next(9, Configuration.maxInputDimension));
Program.Form.AddConfiguration($"\r\nInputDimension: {selectedInputDimension}\r\n");
debug($"function setup done, generating data [inputDimension={selectedInputDimension}] ...");
//Parallel.For(0, Data.ForexPrices.Count / inputDimension, (offset, state) =>
//{
// offset += inputDimension;
//});
for (int currentOffset = 0; currentOffset < Data.Prices.Count && runScan; currentOffset += selectedInputDimension)
{
Program.Form.SetBigLabel($"Generating train/test data ...");
if (currentOffset % 55 == 0)
{
Program.Form.SetStatus(
$"Generating train && test data [{currentOffset} - {currentOffset + selectedInputDimension}] " +
$"{(double) currentOffset / Data.Prices.Count * 100.0,2:0.##}% ...");
}
combinedResult = new double[] { };
foreach (var funct in Data.FunctionConfiguration)
{
if (noDelayEnabled == false)
{
Thread.Sleep(1);
}
var functionInfo = funct.Value;
FunctionParameters functionParameters = new FunctionParameters((MethodInfo)functionInfo["methodInfo"],
selectedInputDimension, currentOffset);
// execute function
Function function = new Function((MethodInfo)functionInfo["methodInfo"]);
result = function.Execute(functionParameters, out var code);
// check function output
if (result == null || result.Length <= 1 || double.IsNegativeInfinity(result[0]) || double.IsPositiveInfinity(result[0]) ||
double.IsNaN(result[0]) || double.IsInfinity(result[0]) || IsArrayRepeating(result))
{
debug($"WARNING: skip {((MethodInfo) functionInfo["methodInfo"]).Name} due to bad output [len={result.Length}, code={code}]");
Program.Form.SetStatus($"ERROR: bad output for {((MethodInfo) functionInfo["methodInfo"]).Name}");
numberOfBrokenData++;
goto again;
}
// copy new output to all data
if (combinedResult != null)
{
prevOffset = combinedResult.Length;
}
Array.Resize(ref combinedResult, (combinedResult?.Length ?? 0) + result.Length);
Array.Copy(result, 0, combinedResult, prevOffset, result.Length);
}
// generate train data set
Array.Resize(ref inputSets, numRecord + 1);
Array.Resize(ref outputSets, numRecord + 1);
inputSets[numRecord] = new double[combinedResult.Length];
outputSets[numRecord] = new double[2];
if (numRecord % 245 == 0)
{
debug(
$"offset: {currentOffset} numRecord:{numRecord} inputSets:{inputSets.Length} outputSets:{outputSets.Length} combinedResult:{combinedResult.Length}");
}
Array.Copy(combinedResult, inputSets[numRecord], combinedResult.Length);
SetOutputResult(selectedInputDimension, currentOffset, numRecord);
numRecord++;
}
TrainNetwork(ref inputSets, ref outputSets);
} while(runScan);
// goto again;
Program.Form.DoingSearch = false;
debug($"done scan: numRecord={numRecord} i:{inputSets?.Length} o:{outputSets?.Length}");
}
/*
* ／ イ(((ヽ
* (ﾉ ￣Ｙ＼
| (＼ (. /) ｜ )
* ヽ ヽ` ( ͡° ͜ʖ ͡°) _ノ /
* ＼ | ⌒Ｙ⌒ / /
* ｜ヽ ｜ ﾉ ／
* ＼トー仝ーイ
* ｜ ミ土彡/
* )\ ° /
* ( \ /
* / / ѼΞΞΞΞΞΞΞD
* / / / \ \ \
* (( ). ) ).)
* ( ). ( | |
| / \ |*/
private void SetupFunctions(int randomSeedLocal)
{
SetStats();
int functionsCount = XRandom.next(Configuration.MinTaFunctionsCount, Configuration.MinTaFunctionsCount + 8);
debug($"selecting functions Count={functionsCount}");
Program.Form.ConfigurationClear();
//Program.Form.AddConfiguration("Functions:\r\n");
Program.Form.EraseBigLabel();
Program.Form.SetBigLabel("[SETTING FUNCTIONS UP]");
for (int i = 0; i < functionsCount && runScan; i++)
{
SetStats();
Program.Form.SetBigLabel($"[SETUP FUNCTION #{i}]");
int unixTimestamp = (int)DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds + DateTime.Now.Millisecond;
// get random method
var selectedMethosdInfo = Methods.GetRandomMethod(unixTimestamp);
Program.Form.SetStatus($"Setup function #{i} <{selectedMethosdInfo.Name}> ...");
debug($"Selected function #{i}: {selectedMethosdInfo.Name} unixTimestamp: {unixTimestamp}");
if (Data.FunctionConfiguration.ContainsKey(selectedMethosdInfo.Name))
{
debug($"function {selectedMethosdInfo.Name} already exist");
if (i > 0)
{
i--;
}
numberOfFailedFunctions++;
continue;
}
// generate parameters
FunctionParameters functionParameters = new FunctionParameters(selectedMethosdInfo, selectedInputDimension, 0);
// execute function
var function = new FinancePermutator.Function(selectedMethosdInfo);
result = function.Execute(functionParameters, out var code);
if (result == null || result.Length <= 1 || double.IsNegativeInfinity(result[0]) || double.IsPositiveInfinity(result[0]) ||
double.IsNaN(result[0]) || double.IsInfinity(result[0]) || IsArrayRepeating(result))
{
DumpValues(selectedMethosdInfo, result);
debug(
$"WARNING: skip {selectedMethosdInfo.Name} due to bad output [len={result.Length}, code={code} InputDimension={selectedInputDimension}], need {Configuration.MinTaFunctionsCount - i}");
if (i > 0)
{
i--;
}
numberOfFailedFunctions++;
continue;
}
//Program.Form.AddConfiguration($" [{methodInfo.Name} \r\n{functionParameters.parametersMap}] \r\n =====================\r\n");
// record info
Data.FunctionConfiguration[selectedMethosdInfo.Name] = new Dictionary <string, object>
{
["parameters"] = functionParameters,
["methodInfo"] = selectedMethosdInfo
};
randomSeedLocal = unixTimestamp + XRandom.next(255);
}
var functions = new StringBuilder();
foreach (var func in Data.FunctionConfiguration)
{
functions.Append($"[{func.Key}] ");
}
Program.Form.funcListLabel.Invoke((MethodInvoker)(() => { Program.Form.funcListLabel.Text = functions.ToString(); }));
//debug($"{JsonConvert.SerializeObject(Data.FunctionBase, Formatting.Indented)}");
//Thread.Sleep(500000);
}
/*░░ ♡ ▄▀▀▀▄░░░
* ▄███▀░◐░░░▌░░░░░░░
* ░░░░▌░░░░░▐░░░░░░░
* ░░░░▐░░░░░▐░░░░░░░
* ░░░░▌░░░░░▐▄▄░░░░░
* ░░░░▌░░░░▄▀▒▒▀▀▀▀▄
* ░░░▐░░░░▐▒▒▒▒▒▒▒▒▀▀▄
* ░░░▐░░░░▐▄▒▒▒▒▒▒▒▒▒▒▀▄
* ░░░░▀▄░░░░▀▄▒▒▒▒▒▒▒▒▒▒▀▄
* ░░░░░░▀▄▄▄▄▄█▄▄▄▄▄▄▄▄▄▄▄▀▄ <<<---.
* ░░░░░░░░░░░▌▌░▌▌░░░░░
* ░░░░░░░░░░░▌▌░▌▌░░░░░
* ░░░░░░░░░▄▄▌▌▄▌▌░░░░░*/
public FunctionParameters(MethodInfo methodInfo, int numdata, int offset)
{
NumData = numdata;
Offset = offset;
parametersMap = new List <string>();
// DumpParams(methodInfo);
Arguments = new object[methodInfo.GetParameters().Length];
// debug($"function method {methodInfo.Name} offset {offset} numdata {NumData} randomSeed {randomSeed}");
foreach (ParameterInfo param in methodInfo.GetParameters())
{
string paramComment = string.Empty;
switch (param.Name)
{
case "optInVFactor":
Arguments[ParamIndex] = 0;
break;
case "outMACDSignal":
Arguments[ParamIndex] = 0;
break;
case "outMACDHist":
Arguments[ParamIndex] = 0;
break;
case "outMin":
Arguments[ParamIndex] = 0;
break;
case "optInNbDevUp":
Arguments[ParamIndex] = 0.0;
break;
case "outMACD":
Arguments[ParamIndex] = 0.0;
break;
case "outLeadSine":
Arguments[ParamIndex] = 0.0;
break;
case "outSine":
Arguments[ParamIndex] = 0.0;
break;
case "optInMinPeriod":
case "optInMaxPeriod":
case "optInSignalPeriod":
Arguments[ParamIndex] = MaTypeGen.GetRandom();
paramComment = $"MaTypeGen";
// debug($"{param.Name} optInSignalPeriod=" + arguments[paramIndex]);
break;
case "optInMAType":
Arguments[ParamIndex] = MaTypeGen.GetRandom();
paramComment = $"MaTypeGen";
// debug($"{param.Name} optInMAType=" + arguments[paramIndex]);
break;
case "optInNbDev":
Arguments[ParamIndex] = 0.0;
break;
case "inReal0":
case "inReal1":
case "inReal":
var index = XRandom.next(3);
switch (index)
{
case 0:
Arguments[ParamIndex] = ForexPrices.GetOpen(NumData, Offset);
paramComment = $"Open {NumData}";
break;
case 1:
Arguments[ParamIndex] = ForexPrices.GetClose(numdata, Offset);
paramComment = $"Close {NumData}";
break;
case 2:
Arguments[ParamIndex] = ForexPrices.GetHigh(NumData, Offset);
paramComment = $"High {NumData}";
break;
case 3:
Arguments[ParamIndex] = ForexPrices.GetLow(NumData, Offset);
paramComment = $"Low {NumData}";
break;
}
// debug($"real {param.Name}[0]: " + ((double[])Arguments[ParamIndex])[0]);
break;
case "optInMaximum":
Arguments[ParamIndex] = 0.0;
break;
case "optInSlowD_MAType":
case "optInFastD_MAType":
case "optInSlowK_MAType":
Arguments[ParamIndex] = MaTypeGen.GetRandom();
paramComment = $"MaTypeGen";
// debug($"{param.Name} optMAtype=" + arguments[paramIndex]);
break;
case "optInAccelerationShort":
case "optInAccelerationMaxShort":
case "optInAccelerationInitShort":
case "optInAccelerationMaxLong":
case "optInAccelerationLong":
case "optInAccelerationInitLong":
case "optInAcceleration":
Arguments[ParamIndex] = 0.0;
break;
case "optInOffsetOnReverse":
Arguments[ParamIndex] = 0;
break;
case "optInSlowK_Period":
case "optInFastK_Period":
case "optInSlowD_Period":
case "optInFastD_Period":
Arguments[ParamIndex] = MaGen.GetRandom(NumData);
paramComment = $"MaTypeGen";
break;
case "optInSlowPeriod":
Arguments[ParamIndex] = XRandom.next(Convert.ToInt32(NumData / 2) + 1, NumData - 1);
break;
case "optInFastPeriod":
Arguments[ParamIndex] = MaGen.GetRandom(Convert.ToInt32(NumData / 2));
break;
case "optInTimePeriod":
Arguments[ParamIndex] = 2;
break;
case "optInTimePeriod1":
case "optInTimePeriod3":
case "optInTimePeriod2":
Arguments[ParamIndex] = MaGen.GetRandom(NumData);
break;
case "optInPenetration":
Arguments[ParamIndex] = 0;
break;
case "optInStartValue":
Arguments[ParamIndex] = 0;
break;
case "startIdx":
Arguments[ParamIndex] = 0;
break;
case "endIdx":
Arguments[ParamIndex] = NumData - 1;
break;
case "inOpen":
Arguments[ParamIndex] = ForexPrices.GetOpen(NumData, Offset);
paramComment = $"Open {NumData}";
break;
case "inHigh":
Arguments[ParamIndex] = ForexPrices.GetHigh(NumData, Offset);
paramComment = $"High {NumData}";
break;
case "inLow":
Arguments[ParamIndex] = ForexPrices.GetLow(NumData, Offset);
paramComment = $"Low {NumData}";
break;
case "inClose":
Arguments[ParamIndex] = ForexPrices.GetClose(NumData, Offset);
paramComment = $"Close {NumData}";
break;
case "inVolume":
Arguments[ParamIndex] = ForexPrices.GetVolume(NumData, Offset);
paramComment = $"Volume {NumData}";
break;
case "outBegIdx":
Arguments[ParamIndex] = this.OutBegIdx;
break;
case "outNBElement":
Arguments[ParamIndex] = OutNbElement;
OutNbElement = ParamIndex;
break;
case "outInteger":
Arguments[ParamIndex] = new int[NumData];
OutIndex = ParamIndex;
break;
case "outReal":
Arguments[ParamIndex] = new double[NumData];
OutIndex = ParamIndex;
break;
case "outAroonUp":
Arguments[ParamIndex] = new double[1000];
break;
case "outAroonDown":
Arguments[ParamIndex] = new double[1000];
break;
case "outSlowD":
case "outSlowK":
case "outFastD":
case "outFastK":
Arguments[ParamIndex] = new double[1000];
break;
default:
Tools.debug($"nothing found for {param.Name}");
break;
}
parametersMap.Add($"{ParamIndex}|{param.Name}|{Arguments[ParamIndex]}|{paramComment}");
ParamIndex++;
}
}