//тест для Format Data
private void buttonFormatTest_Click(object sender, EventArgs e)
{
if (Information.isFormatLearn)
{
openFileDialog1.FileName = "Выбор файла NN";
openFileDialog1.Filter = "*(*.txt)|*.txt";
openFileDialog1.InitialDirectory = Directory.GetCurrentDirectory() + @"\formatData";
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
string filePath = openFileDialog1.FileName;
Information.FormatFilePath = filePath;
}
if (Information.FormatFilePath != "")
{
Tuple <PatternsType, KeysType, AnswersType> data
= FormatLoad(Information.FormatFilePath);
PatternsType pattList = data.Item1;
KeysType keys = data.Item2;
AnswersType answList = data.Item3;
Information.NN.FormatTest(pattList, keys, answList);
}
}
else
{
printErrorMessage("Format Learn Nor Active!");
}
}
private static void Example(int tickCount, KeysType keysType)
{
Stopwatch sw = new Stopwatch();
const string FILE_NAME = "test.stsdb4";
File.Delete(FILE_NAME);
//insert
Console.WriteLine("Inserting...");
sw.Reset();
sw.Start();
int c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table");
foreach (var kv in TicksGenerator.GetFlow(tickCount, keysType)) //generate random records
{
table[kv.Key] = kv.Value;
c++;
if (c % 100000 == 0)
{
Console.WriteLine("Inserted {0} records", c);
}
}
engine.Commit();
}
sw.Stop();
Console.WriteLine("Insert speed:{0} rec/sec", sw.GetSpeed(tickCount));
//read
Console.WriteLine("Reading...");
sw.Reset();
sw.Start();
c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table");
foreach (var row in table) //table.Forward(), table.Backward()
{
//Console.WriteLine("{0} {1}", row.Key, row.Value);
c++;
if (c % 100000 == 0)
{
Console.WriteLine("Read {0} records", c);
}
}
}
sw.Stop();
Console.WriteLine("Read speed:{0} records", sw.GetSpeed(c));
}
public static IEnumerable <KeyValuePair <long, Tick> > GetFlow(long number, KeysType keysType)
{
Random random = new Random((int)DateTime.Now.Ticks);
//init startup prices
var prices = TicksGenerator.prices.ToArray();
DateTime timestamp = DateTime.Now;
//generate ticks
for (long i = 0; i < number; i++)
{
int id = random.Next(symbols.Length);
//random movement (Random Walk)
int direction = random.Next() % 2 == 0 ? 1 : -1;
int pips = random.Next(0, 10);
int spread = random.Next(2, 30);
int seconds = random.Next(1, 30);
string symbol = symbols[id];
int d = digits[id];
double pipSize = pipsizes[id];
//generate values
timestamp = timestamp.AddSeconds(seconds);
double bid = Math.Round(prices[id] + direction * pips * pipSize, d);
double ask = Math.Round(bid + spread * pipSize, d);
int bidSize = random.Next(0, 10000);
int askSize = random.Next(0, 10000);
string provider = providers[random.Next(providers.Length)];
//create tick
Tick tick = new Tick(symbol, timestamp, bid, ask, bidSize, askSize, provider);
long key;
if (keysType == KeysType.Sequential)
{
key = i;
}
else
{
key = unchecked (random.Next() * (i + 1));
}
yield return(new KeyValuePair <long, Tick>(key, tick));
prices[id] = bid;
}
}
private static void Example(int tickCount, KeysType keysType)
{
Stopwatch sw = new Stopwatch();
const string FILE_NAME = "test.stsdb4";
File.Delete(FILE_NAME);
//insert
Console.WriteLine("Inserting...");
sw.Reset();
sw.Start();
int c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable<long, Tick> table = engine.OpenXTable<long, Tick>("table");
foreach (var kv in TicksGenerator.GetFlow(tickCount, keysType)) //generate random records
{
table[kv.Key] = kv.Value;
c++;
if (c % 100000 == 0)
Console.WriteLine("Inserted {0} records", c);
}
engine.Commit();
}
sw.Stop();
Console.WriteLine("Insert speed:{0} rec/sec", sw.GetSpeed(tickCount));
//read
Console.WriteLine("Reading...");
sw.Reset();
sw.Start();
c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable<long, Tick> table = engine.OpenXTable<long, Tick>("table");
foreach (var row in table) //table.Forward(), table.Backward()
{
//Console.WriteLine("{0} {1}", row.Key, row.Value);
c++;
if (c % 100000 == 0)
Console.WriteLine("Read {0} records", c);
}
}
sw.Stop();
Console.WriteLine("Read speed:{0} records", sw.GetSpeed(c));
}
public static IEnumerable<KeyValuePair<long, Tick>> GetFlow(long number, KeysType keysType)
{
Random random = new Random((int)DateTime.Now.Ticks);
//init startup prices
var prices = TicksGenerator.prices.ToArray();
DateTime timestamp = DateTime.Now;
//generate ticks
for (long i = 0; i < number; i++)
{
int id = random.Next(symbols.Length);
//random movement (Random Walk)
int direction = random.Next() % 2 == 0 ? 1 : -1;
int pips = random.Next(0, 10);
int spread = random.Next(2, 30);
int seconds = random.Next(1, 30);
string symbol = symbols[id];
int d = digits[id];
double pipSize = pipsizes[id];
//generate values
timestamp = timestamp.AddSeconds(seconds);
double bid = Math.Round(prices[id] + direction * pips * pipSize, d);
double ask = Math.Round(bid + spread * pipSize, d);
int bidSize = random.Next(0, 10000);
int askSize = random.Next(0, 10000);
string provider = providers[random.Next(providers.Length)];
//create tick
Tick tick = new Tick(symbol, timestamp, bid, ask, bidSize, askSize, provider);
long key;
if (keysType == KeysType.Sequential)
key = i;
else
key = unchecked(random.Next() * (i + 1));
yield return new KeyValuePair<long, Tick>(key, tick);
prices[id] = bid;
}
}
//загрузка Format Data из файла
private Tuple <PatternsType, KeysType, AnswersType> FormatLoad(string filePath)
{
string message = "Данные загружены из ";
message += filePath + "\r\n";
PatternsType patts = new PatternsType();
KeysType keys = new KeysType();
AnswersType answs = new AnswersType();
#region Read Answers
int posLastSlesh = filePath.LastIndexOf('\\');
string answersPath = filePath.Substring(0, posLastSlesh);
answersPath += "\\answers.txt";
answs = LoadVec.LoadFloatVecWithKey(answersPath);
#endregion
#region Read Patterns
string[] readText = File.ReadAllLines(filePath);
foreach (string s in readText)
{
if (s.Trim() != "")
{
string[] elemsStr = s.Split(new char[] { ',' });
List <float> vec = new List <float>(elemsStr.Length - 1);
string key = elemsStr[0];
for (int i = 1; i < elemsStr.Length; i++)
{
float floatEl;
bool ok = float.TryParse(elemsStr[i], out floatEl);
if (ok)
{
vec.Add(floatEl);
}
else
{
FormConsole.PrintlnAndScroll("error try parse");
return(new Tuple <PatternsType, KeysType, AnswersType>(null, null, null));
}
}
patts.Add(vec);
keys.Add(key);
}
}
#endregion
FormConsole.PrintlnAndScroll(message);
return(new Tuple <PatternsType, KeysType, AnswersType>(patts, keys, answs));
}
//Тест количество верных ответов для mnist TestDigits
public void FormatTest(PatternsType patterns, KeysType keys, AnswersType answers)
{
FormConsole.PrintlnAndScroll("\r\nstart Format Test");
int testCount = patterns.Count;
int goodCount = 0;
int[] pattSuccess = new int[answers.Count];
for (int i = 0; i < testCount; i++)
{
List <float> input = patterns[i];
if (IsBiasNeuron && input.Count != this.inputLayer.SizeNeurons)
{
input.Add(1);
}
string label = keys[i];
List <float> output = Run(input, IsBiasNeuron);
if (IsCorrectAnswer(output, answers[label]))
{
goodCount++;
int num = 0;
if (int.TryParse(label, out num))
{
pattSuccess[num]++;
}
}
if (i % 1000 == 0)
{
FormConsole.PrintlnAndScroll("i = " + i + " goodCount = " + goodCount);
}
}
FormConsole.PrintlnAndScroll("goodCount = " + goodCount
+ "\r\nfinish Mnist Test");
for (int i = 0; i < pattSuccess.Length; i++)
{
FormConsole.AppendText(i + ": " + pattSuccess[i].ToString() + "\r\n");
}
}
//Начать учить НН
private async void startLearnNN(object sender, EventArgs e)
{
int MaxEp;
if (int.TryParse(textBoxLearnRate.Text, out MaxEp))
{
Information.NN.MaxEp = MaxEp;
}
int DistPrint;
if (int.TryParse(textBoxMoment.Text, out DistPrint))
{
Information.NN.DistPrint = DistPrint;
}
if (Information.pattNames.Count == Information.answNames.Count &&
Information.pattNames.Count != 0)
{
string[] fileNames1 = Information.pattNames.ToArray();
LoadFromFile lF1 = new LoadFromFile(fileNames1);
string[] fileNames2 = Information.answNames.ToArray();
LoadFromFile lF2 = new LoadFromFile(fileNames2);
List <List <float> > pattList = lF1.ConvertIntoList();
List <List <float> > answList = lF2.ConvertIntoList();
Information.NN.Learning(pattList.Count, pattList, answList);
}
else if (Information.isMnistLearn && Information.answNames.Count == 10)
{
ReadMnist readMnist = new ReadMnist(true, false);
//File.WriteAllLines("mnist_data.txt", Information.mnistStrs);
string[] fileNames2 = Information.answNames.ToArray();
LoadFromFile lF2 = new LoadFromFile(fileNames2);
List <List <float> > answList = lF2.ConvertIntoList();
Information.NN.LearningMnist(readMnist, answList);
}
else if (Information.isFormatLearn)
{
if (Information.NN != null)
{
openFileDialog1.FileName = "Выбор файла NN";
openFileDialog1.Filter = "*(*.txt)|*.txt";
openFileDialog1.InitialDirectory = Directory.GetCurrentDirectory() + @"\formatData";
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
string filePath = openFileDialog1.FileName;
Information.FormatFilePath = filePath;
Tuple <PatternsType, KeysType, AnswersType> data = FormatLoad(filePath);
PatternsType pattList = data.Item1;
KeysType keys = data.Item2;
AnswersType answList = data.Item3;
await Information.NN.LearningFormat(pattList, keys, answList);
}
else
{
FormConsole.PrintlnAndScroll("erroe open file dialog");
}
}
else
{
FormConsole.PrintlnAndScroll("erroe NN is not define");
}
}
else
{
printErrorMessage("Неверно заданы образцы и ответы");
}
}
/// <summary>
/// Generates random flow of rows.
/// </summary>
public static IEnumerable <KeyValuePair <byte[], byte[]> > Flow(this Random random, long COUNT, int KEY_LENGTH, int REC_LENGTH, KeysType keys, ByteOrder byteOrder)
{
if (KEY_LENGTH < 8)
{
throw new ArgumentException("KEY_LENGTH < 8");
}
switch (keys)
{
case KeysType.Random:
{
for (long i = 0; i < COUNT; i++)
{
byte[] key = new byte[KEY_LENGTH];
random.NextBytes(key);
byte[] rec = new byte[REC_LENGTH];
random.NextBytes(rec);
yield return(new KeyValuePair <byte[], byte[]>(key, rec));
}
}
break;
case KeysType.Sequential:
{
switch (byteOrder)
{
case ByteOrder.LittleEndian:
{
for (long i = 0; i < COUNT; i++)
{
byte[] key = new byte[KEY_LENGTH];
var b = BitConverter.GetBytes(i);
key[0] = b[0];
key[1] = b[1];
key[2] = b[2];
key[3] = b[3];
key[4] = b[4];
key[5] = b[5];
key[6] = b[6];
key[7] = b[7];
byte[] rec = new byte[REC_LENGTH];
random.NextBytes(rec);
yield return(new KeyValuePair <byte[], byte[]>(key, rec));
}
}
break;
case ByteOrder.BigEndian:
{
for (long i = 0; i < COUNT; i++)
{
byte[] key = new byte[KEY_LENGTH];
var b = BitConverter.GetBytes(i);
key[KEY_LENGTH - 1] = b[0];
key[KEY_LENGTH - 2] = b[1];
key[KEY_LENGTH - 3] = b[2];
key[KEY_LENGTH - 4] = b[3];
key[KEY_LENGTH - 5] = b[4];
key[KEY_LENGTH - 6] = b[5];
key[KEY_LENGTH - 7] = b[6];
key[KEY_LENGTH - 8] = b[7];
byte[] rec = new byte[REC_LENGTH];
random.NextBytes(rec);
yield return(new KeyValuePair <byte[], byte[]>(key, rec));
}
}
break;
default:
throw new NotSupportedException(byteOrder.ToString());
}
}
break;
default:
throw new NotSupportedException(keys.ToString());
}
}
//обучение с наилучгим форматом
public Task LearningFormat(PatternsType patterns, KeysType keys, AnswersType answers)
{
int setCount = patterns.Count;
System.Windows.Forms.RichTextBox console = FormConsole.console;
console.AppendText("Learning Start\r\n");
Information.form1.Refresh();
if (IsBiasNeuron)
{
for (int i = 0; i < patterns.Count; i++)
{
if (patterns[i].Count != this.inputLayer.SizeNeurons)
{
patterns[i].Add(1);
}
}
}
float epsEpErr = 0.0000001f;
float[] lastErrors = new float[3];
for (int i = 0; i < lastErrors.Length; i++)
{
lastErrors[i] = i;
}
for (int ep = 0; ep < MAX_EP; ep++)
{
DateTime startEpTime = DateTime.Now;
bool isStopLearning = true;
float sumEpErr = 0;
for (int setIter = 0; setIter < setCount; setIter++)
{
List <float> input = patterns[setIter];
string label = keys[setIter];
List <float> output = Run(input, IsBiasNeuron);
float err = ErrorSet.MSE(answers[label], output);
sumEpErr += err;
if (err > 0.03)
{
isStopLearning = false;
}
string outStr = "";
string answStr = "";
foreach (var el in output)
{
outStr += Math.Round(el, 4) + " ";
}
foreach (var el in answers[label])
{
answStr += Math.Round(el, 4) + " ";
}
ChangeWeights(answers[label], output);
if (ep % DIST_PRINT == 0 && setIter % 100 == 0)
{
console.AppendText("ep = " + ep + " setIter = " + setIter
+ " err = " + Math.Round(err, 4) + "\r\n");
console.AppendText("out: " + outStr + "\r\n");
console.AppendText("answ: " + answStr + "\r\n");
console.AppendText("label = " + label + "\r\n");
TimeSpan epTime = DateTime.Now - startEpTime;
console.AppendText("timeEp = " + epTime.ToString("h'h 'm'm 's's'") + "\r\n");
console.SelectionStart = console.TextLength;
console.ScrollToCaret();
Information.form1.Refresh();
}
}
lastErrors[0] = lastErrors[1];
lastErrors[1] = lastErrors[2];
lastErrors[2] = sumEpErr / setCount;
if (Math.Abs(lastErrors[0] - lastErrors[1]) < epsEpErr ||
Math.Abs(lastErrors[1] - lastErrors[2]) < epsEpErr)
{
console.AppendText("exit epsEpErr\r\n");
console.AppendText(lastErrors[0] + "\r\n");
console.AppendText(lastErrors[1] + "\r\n");
console.AppendText(lastErrors[2] + "\r\n");
Information.form1.Refresh();
return(Task.CompletedTask);
}
if (isStopLearning)
{
console.AppendText("exit isStopLearning\r\n");
Information.form1.Refresh();
return(Task.CompletedTask);
}
}
console.AppendText("exit MAX_EP\r\n");
Information.form1.Refresh();
return(Task.CompletedTask);
}
