//тест для Format Data private void buttonFormatTest_Click(object sender, EventArgs e) { if (Information.isFormatLearn) { openFileDialog1.FileName = "Выбор файла NN"; openFileDialog1.Filter = "*(*.txt)|*.txt"; openFileDialog1.InitialDirectory = Directory.GetCurrentDirectory() + @"\formatData"; if (openFileDialog1.ShowDialog() == DialogResult.OK) { string filePath = openFileDialog1.FileName; Information.FormatFilePath = filePath; } if (Information.FormatFilePath != "") { Tuple <PatternsType, KeysType, AnswersType> data = FormatLoad(Information.FormatFilePath); PatternsType pattList = data.Item1; KeysType keys = data.Item2; AnswersType answList = data.Item3; Information.NN.FormatTest(pattList, keys, answList); } } else { printErrorMessage("Format Learn Nor Active!"); } }
private static void Example(int tickCount, KeysType keysType) { Stopwatch sw = new Stopwatch(); const string FILE_NAME = "test.stsdb4"; File.Delete(FILE_NAME); //insert Console.WriteLine("Inserting..."); sw.Reset(); sw.Start(); int c = 0; using (IStorageEngine engine = STSdb.FromFile(FILE_NAME)) { ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table"); foreach (var kv in TicksGenerator.GetFlow(tickCount, keysType)) //generate random records { table[kv.Key] = kv.Value; c++; if (c % 100000 == 0) { Console.WriteLine("Inserted {0} records", c); } } engine.Commit(); } sw.Stop(); Console.WriteLine("Insert speed:{0} rec/sec", sw.GetSpeed(tickCount)); //read Console.WriteLine("Reading..."); sw.Reset(); sw.Start(); c = 0; using (IStorageEngine engine = STSdb.FromFile(FILE_NAME)) { ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table"); foreach (var row in table) //table.Forward(), table.Backward() { //Console.WriteLine("{0} {1}", row.Key, row.Value); c++; if (c % 100000 == 0) { Console.WriteLine("Read {0} records", c); } } } sw.Stop(); Console.WriteLine("Read speed:{0} records", sw.GetSpeed(c)); }
public static IEnumerable <KeyValuePair <long, Tick> > GetFlow(long number, KeysType keysType) { Random random = new Random((int)DateTime.Now.Ticks); //init startup prices var prices = TicksGenerator.prices.ToArray(); DateTime timestamp = DateTime.Now; //generate ticks for (long i = 0; i < number; i++) { int id = random.Next(symbols.Length); //random movement (Random Walk) int direction = random.Next() % 2 == 0 ? 1 : -1; int pips = random.Next(0, 10); int spread = random.Next(2, 30); int seconds = random.Next(1, 30); string symbol = symbols[id]; int d = digits[id]; double pipSize = pipsizes[id]; //generate values timestamp = timestamp.AddSeconds(seconds); double bid = Math.Round(prices[id] + direction * pips * pipSize, d); double ask = Math.Round(bid + spread * pipSize, d); int bidSize = random.Next(0, 10000); int askSize = random.Next(0, 10000); string provider = providers[random.Next(providers.Length)]; //create tick Tick tick = new Tick(symbol, timestamp, bid, ask, bidSize, askSize, provider); long key; if (keysType == KeysType.Sequential) { key = i; } else { key = unchecked (random.Next() * (i + 1)); } yield return(new KeyValuePair <long, Tick>(key, tick)); prices[id] = bid; } }
private static void Example(int tickCount, KeysType keysType) { Stopwatch sw = new Stopwatch(); const string FILE_NAME = "test.stsdb4"; File.Delete(FILE_NAME); //insert Console.WriteLine("Inserting..."); sw.Reset(); sw.Start(); int c = 0; using (IStorageEngine engine = STSdb.FromFile(FILE_NAME)) { ITable<long, Tick> table = engine.OpenXTable<long, Tick>("table"); foreach (var kv in TicksGenerator.GetFlow(tickCount, keysType)) //generate random records { table[kv.Key] = kv.Value; c++; if (c % 100000 == 0) Console.WriteLine("Inserted {0} records", c); } engine.Commit(); } sw.Stop(); Console.WriteLine("Insert speed:{0} rec/sec", sw.GetSpeed(tickCount)); //read Console.WriteLine("Reading..."); sw.Reset(); sw.Start(); c = 0; using (IStorageEngine engine = STSdb.FromFile(FILE_NAME)) { ITable<long, Tick> table = engine.OpenXTable<long, Tick>("table"); foreach (var row in table) //table.Forward(), table.Backward() { //Console.WriteLine("{0} {1}", row.Key, row.Value); c++; if (c % 100000 == 0) Console.WriteLine("Read {0} records", c); } } sw.Stop(); Console.WriteLine("Read speed:{0} records", sw.GetSpeed(c)); }
public static IEnumerable<KeyValuePair<long, Tick>> GetFlow(long number, KeysType keysType) { Random random = new Random((int)DateTime.Now.Ticks); //init startup prices var prices = TicksGenerator.prices.ToArray(); DateTime timestamp = DateTime.Now; //generate ticks for (long i = 0; i < number; i++) { int id = random.Next(symbols.Length); //random movement (Random Walk) int direction = random.Next() % 2 == 0 ? 1 : -1; int pips = random.Next(0, 10); int spread = random.Next(2, 30); int seconds = random.Next(1, 30); string symbol = symbols[id]; int d = digits[id]; double pipSize = pipsizes[id]; //generate values timestamp = timestamp.AddSeconds(seconds); double bid = Math.Round(prices[id] + direction * pips * pipSize, d); double ask = Math.Round(bid + spread * pipSize, d); int bidSize = random.Next(0, 10000); int askSize = random.Next(0, 10000); string provider = providers[random.Next(providers.Length)]; //create tick Tick tick = new Tick(symbol, timestamp, bid, ask, bidSize, askSize, provider); long key; if (keysType == KeysType.Sequential) key = i; else key = unchecked(random.Next() * (i + 1)); yield return new KeyValuePair<long, Tick>(key, tick); prices[id] = bid; } }
//загрузка Format Data из файла private Tuple <PatternsType, KeysType, AnswersType> FormatLoad(string filePath) { string message = "Данные загружены из "; message += filePath + "\r\n"; PatternsType patts = new PatternsType(); KeysType keys = new KeysType(); AnswersType answs = new AnswersType(); #region Read Answers int posLastSlesh = filePath.LastIndexOf('\\'); string answersPath = filePath.Substring(0, posLastSlesh); answersPath += "\\answers.txt"; answs = LoadVec.LoadFloatVecWithKey(answersPath); #endregion #region Read Patterns string[] readText = File.ReadAllLines(filePath); foreach (string s in readText) { if (s.Trim() != "") { string[] elemsStr = s.Split(new char[] { ',' }); List <float> vec = new List <float>(elemsStr.Length - 1); string key = elemsStr[0]; for (int i = 1; i < elemsStr.Length; i++) { float floatEl; bool ok = float.TryParse(elemsStr[i], out floatEl); if (ok) { vec.Add(floatEl); } else { FormConsole.PrintlnAndScroll("error try parse"); return(new Tuple <PatternsType, KeysType, AnswersType>(null, null, null)); } } patts.Add(vec); keys.Add(key); } } #endregion FormConsole.PrintlnAndScroll(message); return(new Tuple <PatternsType, KeysType, AnswersType>(patts, keys, answs)); }
//Тест количество верных ответов для mnist TestDigits public void FormatTest(PatternsType patterns, KeysType keys, AnswersType answers) { FormConsole.PrintlnAndScroll("\r\nstart Format Test"); int testCount = patterns.Count; int goodCount = 0; int[] pattSuccess = new int[answers.Count]; for (int i = 0; i < testCount; i++) { List <float> input = patterns[i]; if (IsBiasNeuron && input.Count != this.inputLayer.SizeNeurons) { input.Add(1); } string label = keys[i]; List <float> output = Run(input, IsBiasNeuron); if (IsCorrectAnswer(output, answers[label])) { goodCount++; int num = 0; if (int.TryParse(label, out num)) { pattSuccess[num]++; } } if (i % 1000 == 0) { FormConsole.PrintlnAndScroll("i = " + i + " goodCount = " + goodCount); } } FormConsole.PrintlnAndScroll("goodCount = " + goodCount + "\r\nfinish Mnist Test"); for (int i = 0; i < pattSuccess.Length; i++) { FormConsole.AppendText(i + ": " + pattSuccess[i].ToString() + "\r\n"); } }
//Начать учить НН private async void startLearnNN(object sender, EventArgs e) { int MaxEp; if (int.TryParse(textBoxLearnRate.Text, out MaxEp)) { Information.NN.MaxEp = MaxEp; } int DistPrint; if (int.TryParse(textBoxMoment.Text, out DistPrint)) { Information.NN.DistPrint = DistPrint; } if (Information.pattNames.Count == Information.answNames.Count && Information.pattNames.Count != 0) { string[] fileNames1 = Information.pattNames.ToArray(); LoadFromFile lF1 = new LoadFromFile(fileNames1); string[] fileNames2 = Information.answNames.ToArray(); LoadFromFile lF2 = new LoadFromFile(fileNames2); List <List <float> > pattList = lF1.ConvertIntoList(); List <List <float> > answList = lF2.ConvertIntoList(); Information.NN.Learning(pattList.Count, pattList, answList); } else if (Information.isMnistLearn && Information.answNames.Count == 10) { ReadMnist readMnist = new ReadMnist(true, false); //File.WriteAllLines("mnist_data.txt", Information.mnistStrs); string[] fileNames2 = Information.answNames.ToArray(); LoadFromFile lF2 = new LoadFromFile(fileNames2); List <List <float> > answList = lF2.ConvertIntoList(); Information.NN.LearningMnist(readMnist, answList); } else if (Information.isFormatLearn) { if (Information.NN != null) { openFileDialog1.FileName = "Выбор файла NN"; openFileDialog1.Filter = "*(*.txt)|*.txt"; openFileDialog1.InitialDirectory = Directory.GetCurrentDirectory() + @"\formatData"; if (openFileDialog1.ShowDialog() == DialogResult.OK) { string filePath = openFileDialog1.FileName; Information.FormatFilePath = filePath; Tuple <PatternsType, KeysType, AnswersType> data = FormatLoad(filePath); PatternsType pattList = data.Item1; KeysType keys = data.Item2; AnswersType answList = data.Item3; await Information.NN.LearningFormat(pattList, keys, answList); } else { FormConsole.PrintlnAndScroll("erroe open file dialog"); } } else { FormConsole.PrintlnAndScroll("erroe NN is not define"); } } else { printErrorMessage("Неверно заданы образцы и ответы"); } }
/// <summary> /// Generates random flow of rows. /// </summary> public static IEnumerable <KeyValuePair <byte[], byte[]> > Flow(this Random random, long COUNT, int KEY_LENGTH, int REC_LENGTH, KeysType keys, ByteOrder byteOrder) { if (KEY_LENGTH < 8) { throw new ArgumentException("KEY_LENGTH < 8"); } switch (keys) { case KeysType.Random: { for (long i = 0; i < COUNT; i++) { byte[] key = new byte[KEY_LENGTH]; random.NextBytes(key); byte[] rec = new byte[REC_LENGTH]; random.NextBytes(rec); yield return(new KeyValuePair <byte[], byte[]>(key, rec)); } } break; case KeysType.Sequential: { switch (byteOrder) { case ByteOrder.LittleEndian: { for (long i = 0; i < COUNT; i++) { byte[] key = new byte[KEY_LENGTH]; var b = BitConverter.GetBytes(i); key[0] = b[0]; key[1] = b[1]; key[2] = b[2]; key[3] = b[3]; key[4] = b[4]; key[5] = b[5]; key[6] = b[6]; key[7] = b[7]; byte[] rec = new byte[REC_LENGTH]; random.NextBytes(rec); yield return(new KeyValuePair <byte[], byte[]>(key, rec)); } } break; case ByteOrder.BigEndian: { for (long i = 0; i < COUNT; i++) { byte[] key = new byte[KEY_LENGTH]; var b = BitConverter.GetBytes(i); key[KEY_LENGTH - 1] = b[0]; key[KEY_LENGTH - 2] = b[1]; key[KEY_LENGTH - 3] = b[2]; key[KEY_LENGTH - 4] = b[3]; key[KEY_LENGTH - 5] = b[4]; key[KEY_LENGTH - 6] = b[5]; key[KEY_LENGTH - 7] = b[6]; key[KEY_LENGTH - 8] = b[7]; byte[] rec = new byte[REC_LENGTH]; random.NextBytes(rec); yield return(new KeyValuePair <byte[], byte[]>(key, rec)); } } break; default: throw new NotSupportedException(byteOrder.ToString()); } } break; default: throw new NotSupportedException(keys.ToString()); } }
//обучение с наилучгим форматом public Task LearningFormat(PatternsType patterns, KeysType keys, AnswersType answers) { int setCount = patterns.Count; System.Windows.Forms.RichTextBox console = FormConsole.console; console.AppendText("Learning Start\r\n"); Information.form1.Refresh(); if (IsBiasNeuron) { for (int i = 0; i < patterns.Count; i++) { if (patterns[i].Count != this.inputLayer.SizeNeurons) { patterns[i].Add(1); } } } float epsEpErr = 0.0000001f; float[] lastErrors = new float[3]; for (int i = 0; i < lastErrors.Length; i++) { lastErrors[i] = i; } for (int ep = 0; ep < MAX_EP; ep++) { DateTime startEpTime = DateTime.Now; bool isStopLearning = true; float sumEpErr = 0; for (int setIter = 0; setIter < setCount; setIter++) { List <float> input = patterns[setIter]; string label = keys[setIter]; List <float> output = Run(input, IsBiasNeuron); float err = ErrorSet.MSE(answers[label], output); sumEpErr += err; if (err > 0.03) { isStopLearning = false; } string outStr = ""; string answStr = ""; foreach (var el in output) { outStr += Math.Round(el, 4) + " "; } foreach (var el in answers[label]) { answStr += Math.Round(el, 4) + " "; } ChangeWeights(answers[label], output); if (ep % DIST_PRINT == 0 && setIter % 100 == 0) { console.AppendText("ep = " + ep + " setIter = " + setIter + " err = " + Math.Round(err, 4) + "\r\n"); console.AppendText("out: " + outStr + "\r\n"); console.AppendText("answ: " + answStr + "\r\n"); console.AppendText("label = " + label + "\r\n"); TimeSpan epTime = DateTime.Now - startEpTime; console.AppendText("timeEp = " + epTime.ToString("h'h 'm'm 's's'") + "\r\n"); console.SelectionStart = console.TextLength; console.ScrollToCaret(); Information.form1.Refresh(); } } lastErrors[0] = lastErrors[1]; lastErrors[1] = lastErrors[2]; lastErrors[2] = sumEpErr / setCount; if (Math.Abs(lastErrors[0] - lastErrors[1]) < epsEpErr || Math.Abs(lastErrors[1] - lastErrors[2]) < epsEpErr) { console.AppendText("exit epsEpErr\r\n"); console.AppendText(lastErrors[0] + "\r\n"); console.AppendText(lastErrors[1] + "\r\n"); console.AppendText(lastErrors[2] + "\r\n"); Information.form1.Refresh(); return(Task.CompletedTask); } if (isStopLearning) { console.AppendText("exit isStopLearning\r\n"); Information.form1.Refresh(); return(Task.CompletedTask); } } console.AppendText("exit MAX_EP\r\n"); Information.form1.Refresh(); return(Task.CompletedTask); }