public static BasicMLDataSet CreateEvaluationSetAndLoad(string @fileName, int startLine, int HowMany, int WindowSize, int outputsize) { List <double> Opens = QuickCSVUtils.QuickParseCSV(fileName, "Open", startLine, HowMany); List <double> High = QuickCSVUtils.QuickParseCSV(fileName, "High", startLine, HowMany); // List<double> Low = QuickCSVUtils.QuickParseCSV(fileName, "Low", startLine, HowMany); List <double> Close = QuickCSVUtils.QuickParseCSV(fileName, "Close", startLine, HowMany); List <double> Volume = QuickCSVUtils.QuickParseCSV(fileName, 5, startLine, HowMany); double[] Ranges = NetworkUtility.CalculateRanges(Opens.ToArray(), Close.ToArray()); IMLDataPair aPairInput = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Opens.ToArray()), NetworkUtility.CalculatePercents(Opens.ToArray()), WindowSize, outputsize); IMLDataPair aPairInput3 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Close.ToArray()), NetworkUtility.CalculatePercents(Close.ToArray()), WindowSize, outputsize); IMLDataPair aPairInput2 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(High.ToArray()), NetworkUtility.CalculatePercents(High.ToArray()), WindowSize, outputsize); IMLDataPair aPairInput4 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Volume.ToArray()), NetworkUtility.CalculatePercents(Volume.ToArray()), WindowSize, outputsize); IMLDataPair aPairInput5 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Ranges.ToArray()), NetworkUtility.CalculatePercents(Ranges.ToArray()), WindowSize, outputsize); List <IMLDataPair> listData = new List <IMLDataPair>(); listData.Add(aPairInput); listData.Add(aPairInput2); listData.Add(aPairInput3); listData.Add(aPairInput4); listData.Add((aPairInput5)); var minitrainning = new BasicMLDataSet(listData); return(minitrainning); }
public JsonResult SetMark(long userId, UserLanguages userLanguages, KnowledgeMark mark, TrainerItem item, RepetitionType repetitionType = RepetitionType.All) { var itemDataType = (KnowledgeDataType)(item != null ? item.DataType : int.MinValue); if (IsInvalid(userId, userLanguages) || EnumValidator.IsInvalid(mark) || item == null || IdValidator.IsInvalid(item.DataId) || EnumValidator.IsInvalid(itemDataType) || EnumValidator.IsInvalid(repetitionType)) { //TODO: писать сообщение return(JsonResultHelper.Error()); } IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId, repetitionType); var repetitionInterval = new UserRepetitionIntervalItem { DataId = item.DataId, DataType = itemDataType, SourceType = KnowledgeSourceType }; var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages); return(trainerHelper.SetMark(Request, mark, repetitionInterval)); }
protected ActionResult GetIndex(long userId, UserLanguages userLanguages, long groupId, Action <TrainerModel> modelSetter) { if (WebSettingsConfig.Instance.IsSectionForbidden(SectionId)) { return(RedirectToAction("Index", RouteConfig.MAIN_CONTROLLER_NAME)); } if (IdValidator.IsInvalid(userId) || UserLanguages.IsInvalid(userLanguages) || IdValidator.IsInvalid(groupId)) { return(RedirectToParentPage()); } IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId, groupId); var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages); TrainerModel model = trainerHelper.GetTrainerModel(Request); modelSetter(model); if (EnumerableValidator.IsNullOrEmpty(model.Items)) { return(RedirectToParentPage()); } ViewData[OurHtmlHelper.ViewDataFlags.SKIP_POPUP_PANEL] = true; return(View("../Trainer/Index", model)); }
public static TemporalMLDataSet GenerateATemporalSet(string @fileName, int startLine, int HowMany, int WindowSize, int outputsize) { List <double> Opens = QuickCSVUtils.QuickParseCSV(fileName, "Open", startLine, HowMany); List <double> High = QuickCSVUtils.QuickParseCSV(fileName, "High", startLine, HowMany); List <double> Low = QuickCSVUtils.QuickParseCSV(fileName, "Low", startLine, HowMany); List <double> Close = QuickCSVUtils.QuickParseCSV(fileName, "Close", startLine, HowMany); List <double> Volume = QuickCSVUtils.QuickParseCSV(fileName, 5, startLine, HowMany); return(TrainerHelper.GenerateTrainingWithPercentChangeOnSerie(WindowSize, outputsize, Opens.ToArray(), Close.ToArray(), High.ToArray(), Low.ToArray(), Volume.ToArray())); }
private List <TrainerItem> GetOverdueItems(long userId, UserLanguages userLanguages) { IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId, RepetitionType. Overdue); var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages); TrainerModel model = trainerHelper.GetTrainerModel(Request); List <TrainerItem> items = model.Items; return(items); }
// ReSharper disable UnusedMember.Local private static void CreateEvaluationSet(string @fileName) // ReSharper restore UnusedMember.Local { List <double> Opens = QuickCSVUtils.QuickParseCSV(fileName, "Open", 1200, 1200); List <double> High = QuickCSVUtils.QuickParseCSV(fileName, "High", 1200, 1200); List <double> Low = QuickCSVUtils.QuickParseCSV(fileName, "Low", 1200, 1200); List <double> Close = QuickCSVUtils.QuickParseCSV(fileName, "Close", 1200, 1200); List <double> Volume = QuickCSVUtils.QuickParseCSV(fileName, 5, 1200, 1200); double[] Ranges = NetworkUtility.CalculateRanges(Opens.ToArray(), Close.ToArray()); TemporalMLDataSet superTemportal = TrainerHelper.GenerateTrainingWithPercentChangeOnSerie(100, 1, Opens.ToArray(), Close.ToArray(), High.ToArray(), Low.ToArray(), Volume.ToArray()); IMLDataPair aPairInput = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Opens.ToArray()), NetworkUtility.CalculatePercents(Opens.ToArray()), 100, 1); IMLDataPair aPairInput3 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Close.ToArray()), NetworkUtility.CalculatePercents(Close.ToArray()), 100, 1); IMLDataPair aPairInput2 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(High.ToArray()), NetworkUtility.CalculatePercents(High.ToArray()), 100, 1); IMLDataPair aPairInput4 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Volume.ToArray()), NetworkUtility.CalculatePercents(Volume.ToArray()), 100, 1); IMLDataPair aPairInput5 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Ranges.ToArray()), NetworkUtility.CalculatePercents(Ranges.ToArray()), 100, 1); List <IMLDataPair> listData = new List <IMLDataPair>(); listData.Add(aPairInput); listData.Add(aPairInput2); listData.Add(aPairInput3); listData.Add(aPairInput4); listData.Add((aPairInput5)); var minitrainning = new BasicMLDataSet(listData); var network = (BasicNetwork)CreateElmanNetwork(100, 1); double normalCorrectRate = EvaluateNetworks(network, minitrainning); double temporalErrorRate = EvaluateNetworks(network, superTemportal); Console.WriteLine(@"Percent Correct with normal Data Set:" + normalCorrectRate + @" Percent Correct with temporal Dataset:" + temporalErrorRate); Console.WriteLine(@"Paused , Press a key to continue to evaluation"); Console.ReadKey(); }
TVectorPredictor Train(RoleMappedData data) { Contracts.CheckValue(data, "data"); data.CheckFeatureFloatVector(); int count; data.CheckMulticlassLabel(out count); using (var ch = Host.Start("Training")) { // Train one-vs-all models. _predictors = new TVectorPredictor[1]; for (int i = 0; i < _predictors.Length; i++) { ch.Info("Training learner {0}", i); Contracts.CheckValue(_args.predictorType, "predictorType", "Must specify a base learner type"); TScalarTrainer trainer; if (_trainer != null) { trainer = _trainer; } else { var temp = ScikitSubComponent <ITrainer, SignatureBinaryClassifierTrainer> .AsSubComponent(_args.predictorType); var inst = temp.CreateInstance(Host); trainer = inst as TScalarTrainer; if (trainer == null) { var allTypes = TrainerHelper.GetParentTypes(inst.GetType()).ToArray(); var allTypesStr = string.Join("\n", allTypes.Select(c => c.ToString())); throw ch.ExceptNotSupp($"Unable to cast {inst.GetType()} into {typeof(TScalarTrainer)}.\n-TYPES-\n{allTypesStr}"); } } _trainer = null; _predictors[i] = TrainPredictor(ch, trainer, data, count); } } return(CreatePredictor()); }
public ActionResult Index(long userId, UserLanguages userLanguages) { if (WebSettingsConfig.Instance.IsSectionForbidden(SectionId.MyKnowledge)) { return(RedirectToAction("Index", RouteConfig.MAIN_CONTROLLER_NAME)); } if (IsInvalid(userId, userLanguages)) { return(RedirectToKnowledgeWall()); } IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId, RepetitionType. All); var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages); TrainerModel model = trainerHelper.GetTrainerModel(Request); var pageRequiredData = new PageRequiredData(SectionId.MyKnowledge); pageRequiredData.Title = WebSettingsConfig.Instance.GetTemplateText(pageRequiredData.SectionId, PageId.Cards, TemplateId.Title, PageName); pageRequiredData.Keywords = WebSettingsConfig.Instance.GetTemplateText(pageRequiredData.SectionId, PageId.Cards, TemplateId.Keywords, PageName); pageRequiredData.Description = WebSettingsConfig.Instance.GetTemplateText(pageRequiredData.SectionId, PageId.Cards, TemplateId.Description, PageName); model.PageRequiredData = pageRequiredData; model.SetMarkUrl = Url.Action("SetMark", ControllerName, null, Request.Url.Scheme); model.BreadcrumbsItems = new List <BreadcrumbItem> { new BreadcrumbItem { Title = "Стена знаний", Action = "Index", ControllerName = RouteConfig.USER_KNOWLEDGE_CONTROLLER }, new BreadcrumbItem { IsActive = true, Title = PageName } }; if (EnumerableValidator.IsNullOrEmpty(model.Items)) { return(RedirectToKnowledgeWall()); } ViewData[OurHtmlHelper.ViewDataFlags.SKIP_POPUP_PANEL] = true; return(View("../Trainer/Index", model)); }
/// <summary> /// Trains a random trainer. /// </summary> /// <param name="inputs">The inputs.</param> /// <param name="predictWindow">The predict window.</param> public static double RandomTrainerMethod(int inputs, int predictWindow) { double[] firstinput = MakeInputs(inputs); double[] SecondInput = MakeInputs(inputs); double[] ThirdInputs = MakeInputs(inputs); double[] FourthInputs = MakeInputs(inputs); double[] inp5 = MakeInputs(inputs); double[] inp6 = MakeInputs(inputs); var pair = TrainerHelper.ProcessPairs(firstinput, firstinput, inputs, predictWindow); var pair2 = TrainerHelper.ProcessPairs(SecondInput, firstinput, inputs, predictWindow); var pair3 = TrainerHelper.ProcessPairs(ThirdInputs, firstinput, inputs, predictWindow); var pair4 = TrainerHelper.ProcessPairs(FourthInputs, firstinput, inputs, predictWindow); var pair5 = TrainerHelper.ProcessPairs(inp5, firstinput, inputs, predictWindow); var pair6 = TrainerHelper.ProcessPairs(inp6, firstinput, inputs, predictWindow); BasicMLDataSet SuperSet = new BasicMLDataSet(); SuperSet.Add(pair); SuperSet.Add(pair2); SuperSet.Add(pair3); SuperSet.Add(pair4); var network = new BasicNetwork(); network.AddLayer(new BasicLayer(new ActivationTANH(), true, SuperSet.InputSize)); network.AddLayer(new BasicLayer(new ActivationTANH(), false, 20)); network.AddLayer(new BasicLayer(new ActivationTANH(), true, 0)); network.AddLayer(new BasicLayer(new ActivationLinear(), true, predictWindow)); //var layer = new BasicLayer(new ActivationTANH(), true, SuperSet.InputSize); //layer.Network = network; network.Structure.FinalizeStructure(); network.Reset(); // var network = (BasicNetwork)CreateEval.CreateElmanNetwork(SuperSet.InputSize, SuperSet.IdealSize); return(CreateEval.TrainNetworks(network, SuperSet)); //Lets create an evaluation. //Console.WriteLine(@"Last error rate on random trainer:" + error); }
/// <summary> /// Trains the pipeline with data coming from a <see cref="IDataView"/>. /// </summary> public ScikitPipeline Train(IDataView data, string feature = "Feature", string label = null, string weight = null, string groupId = null) { IDataView trans = data; using (var ch = _env.Start("Create transforms")) { for (int i = 0; i < _transforms.Length; ++i) { try { trans = _env.CreateTransform(_transforms[i].transformSettings, trans); } catch (Exception e) { if (e.ToString().Contains("Unknown loadable class")) { var nn = _env.ComponentCatalog.GetAllClasses().Length; var filt = _env.ComponentCatalog.GetAllClasses() .Select(c => c.UserName) .OrderBy(c => c) .Where(c => c.Trim().Length > 2); var regis = string.Join("\n", filt); throw Contracts.Except(e, $"Unable to create transform '{_transforms[i].transformSettings}', assembly not registered among {nn}\n{regis}"); } throw e; } _transforms[i].transform = trans as IDataTransform; } } if (_predictor != null) { using (var ch = _env.Start("Create Predictor")) { _predictor.trainer = TrainerHelper.CreateTrainer(_env, _predictor.trainerSettings); _roles = new List <KeyValuePair <RoleMappedSchema.ColumnRole, string> >(); _roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Feature, feature)); if (!string.IsNullOrEmpty(label)) { _roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Label, label)); } if (!string.IsNullOrEmpty(groupId)) { _roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Group, groupId)); } if (!string.IsNullOrEmpty(weight)) { _roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Weight, weight)); } var roleMap = new RoleMappedData(trans, label, feature, group: groupId, weight: weight); _predictor.predictor = _predictor.trainer.Train(_env, ch, roleMap); _predictor.roleMapData = roleMap; } } else { _predictor = new StepPredictor() { predictor = null, trainer = null, trainerSettings = null, roleMapData = new RoleMappedData(trans) }; // We predict one to make sure everything works fine. using (var ch = _env.Start("Compute one prediction.")) { var df = DataFrameIO.ReadView(trans, 1, keepVectors: true, env: _env); if (df.Length == 0) { throw _env.ExceptEmpty("Something went wrong. The pipeline does not produce any output."); } } } return(this); }