public static BasicMLDataSet CreateEvaluationSetAndLoad(string @fileName, int startLine, int HowMany, int WindowSize, int outputsize)
{
List <double> Opens = QuickCSVUtils.QuickParseCSV(fileName, "Open", startLine, HowMany);
List <double> High = QuickCSVUtils.QuickParseCSV(fileName, "High", startLine, HowMany);
// List<double> Low = QuickCSVUtils.QuickParseCSV(fileName, "Low", startLine, HowMany);
List <double> Close = QuickCSVUtils.QuickParseCSV(fileName, "Close", startLine, HowMany);
List <double> Volume = QuickCSVUtils.QuickParseCSV(fileName, 5, startLine, HowMany);
double[] Ranges = NetworkUtility.CalculateRanges(Opens.ToArray(), Close.ToArray());
IMLDataPair aPairInput = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Opens.ToArray()), NetworkUtility.CalculatePercents(Opens.ToArray()), WindowSize, outputsize);
IMLDataPair aPairInput3 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Close.ToArray()), NetworkUtility.CalculatePercents(Close.ToArray()), WindowSize, outputsize);
IMLDataPair aPairInput2 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(High.ToArray()), NetworkUtility.CalculatePercents(High.ToArray()), WindowSize, outputsize);
IMLDataPair aPairInput4 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Volume.ToArray()), NetworkUtility.CalculatePercents(Volume.ToArray()), WindowSize, outputsize);
IMLDataPair aPairInput5 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Ranges.ToArray()), NetworkUtility.CalculatePercents(Ranges.ToArray()), WindowSize, outputsize);
List <IMLDataPair> listData = new List <IMLDataPair>();
listData.Add(aPairInput);
listData.Add(aPairInput2);
listData.Add(aPairInput3);
listData.Add(aPairInput4);
listData.Add((aPairInput5));
var minitrainning = new BasicMLDataSet(listData);
return(minitrainning);
}
public JsonResult SetMark(long userId,
UserLanguages userLanguages,
KnowledgeMark mark,
TrainerItem item,
RepetitionType repetitionType = RepetitionType.All)
{
var itemDataType = (KnowledgeDataType)(item != null ? item.DataType : int.MinValue);
if (IsInvalid(userId, userLanguages) || EnumValidator.IsInvalid(mark) || item == null ||
IdValidator.IsInvalid(item.DataId) || EnumValidator.IsInvalid(itemDataType) ||
EnumValidator.IsInvalid(repetitionType))
{
//TODO: писать сообщение
return(JsonResultHelper.Error());
}
IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId,
repetitionType);
var repetitionInterval = new UserRepetitionIntervalItem
{
DataId = item.DataId, DataType = itemDataType, SourceType = KnowledgeSourceType
};
var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages);
return(trainerHelper.SetMark(Request, mark, repetitionInterval));
}
protected ActionResult GetIndex(long userId,
UserLanguages userLanguages,
long groupId,
Action <TrainerModel> modelSetter)
{
if (WebSettingsConfig.Instance.IsSectionForbidden(SectionId))
{
return(RedirectToAction("Index", RouteConfig.MAIN_CONTROLLER_NAME));
}
if (IdValidator.IsInvalid(userId) || UserLanguages.IsInvalid(userLanguages) ||
IdValidator.IsInvalid(groupId))
{
return(RedirectToParentPage());
}
IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId, groupId);
var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages);
TrainerModel model = trainerHelper.GetTrainerModel(Request);
modelSetter(model);
if (EnumerableValidator.IsNullOrEmpty(model.Items))
{
return(RedirectToParentPage());
}
ViewData[OurHtmlHelper.ViewDataFlags.SKIP_POPUP_PANEL] = true;
return(View("../Trainer/Index", model));
}
public static TemporalMLDataSet GenerateATemporalSet(string @fileName, int startLine, int HowMany, int WindowSize, int outputsize)
{
List <double> Opens = QuickCSVUtils.QuickParseCSV(fileName, "Open", startLine, HowMany);
List <double> High = QuickCSVUtils.QuickParseCSV(fileName, "High", startLine, HowMany);
List <double> Low = QuickCSVUtils.QuickParseCSV(fileName, "Low", startLine, HowMany);
List <double> Close = QuickCSVUtils.QuickParseCSV(fileName, "Close", startLine, HowMany);
List <double> Volume = QuickCSVUtils.QuickParseCSV(fileName, 5, startLine, HowMany);
return(TrainerHelper.GenerateTrainingWithPercentChangeOnSerie(WindowSize, outputsize, Opens.ToArray(), Close.ToArray(), High.ToArray(), Low.ToArray(), Volume.ToArray()));
}
private List <TrainerItem> GetOverdueItems(long userId, UserLanguages userLanguages)
{
IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId,
RepetitionType.
Overdue);
var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages);
TrainerModel model = trainerHelper.GetTrainerModel(Request);
List <TrainerItem> items = model.Items;
return(items);
}
// ReSharper disable UnusedMember.Local
private static void CreateEvaluationSet(string @fileName)
// ReSharper restore UnusedMember.Local
{
List <double> Opens = QuickCSVUtils.QuickParseCSV(fileName, "Open", 1200, 1200);
List <double> High = QuickCSVUtils.QuickParseCSV(fileName, "High", 1200, 1200);
List <double> Low = QuickCSVUtils.QuickParseCSV(fileName, "Low", 1200, 1200);
List <double> Close = QuickCSVUtils.QuickParseCSV(fileName, "Close", 1200, 1200);
List <double> Volume = QuickCSVUtils.QuickParseCSV(fileName, 5, 1200, 1200);
double[] Ranges = NetworkUtility.CalculateRanges(Opens.ToArray(), Close.ToArray());
TemporalMLDataSet superTemportal = TrainerHelper.GenerateTrainingWithPercentChangeOnSerie(100, 1, Opens.ToArray(),
Close.ToArray(), High.ToArray(), Low.ToArray(), Volume.ToArray());
IMLDataPair aPairInput = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Opens.ToArray()), NetworkUtility.CalculatePercents(Opens.ToArray()), 100, 1);
IMLDataPair aPairInput3 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Close.ToArray()), NetworkUtility.CalculatePercents(Close.ToArray()), 100, 1);
IMLDataPair aPairInput2 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(High.ToArray()), NetworkUtility.CalculatePercents(High.ToArray()), 100, 1);
IMLDataPair aPairInput4 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Volume.ToArray()), NetworkUtility.CalculatePercents(Volume.ToArray()), 100, 1);
IMLDataPair aPairInput5 = TrainerHelper.ProcessPairs(NetworkUtility.CalculatePercents(Ranges.ToArray()), NetworkUtility.CalculatePercents(Ranges.ToArray()), 100, 1);
List <IMLDataPair> listData = new List <IMLDataPair>();
listData.Add(aPairInput);
listData.Add(aPairInput2);
listData.Add(aPairInput3);
listData.Add(aPairInput4);
listData.Add((aPairInput5));
var minitrainning = new BasicMLDataSet(listData);
var network = (BasicNetwork)CreateElmanNetwork(100, 1);
double normalCorrectRate = EvaluateNetworks(network, minitrainning);
double temporalErrorRate = EvaluateNetworks(network, superTemportal);
Console.WriteLine(@"Percent Correct with normal Data Set:" + normalCorrectRate + @" Percent Correct with temporal Dataset:" +
temporalErrorRate);
Console.WriteLine(@"Paused , Press a key to continue to evaluation");
Console.ReadKey();
}
TVectorPredictor Train(RoleMappedData data)
{
Contracts.CheckValue(data, "data");
data.CheckFeatureFloatVector();
int count;
data.CheckMulticlassLabel(out count);
using (var ch = Host.Start("Training"))
{
// Train one-vs-all models.
_predictors = new TVectorPredictor[1];
for (int i = 0; i < _predictors.Length; i++)
{
ch.Info("Training learner {0}", i);
Contracts.CheckValue(_args.predictorType, "predictorType", "Must specify a base learner type");
TScalarTrainer trainer;
if (_trainer != null)
{
trainer = _trainer;
}
else
{
var temp = ScikitSubComponent <ITrainer, SignatureBinaryClassifierTrainer> .AsSubComponent(_args.predictorType);
var inst = temp.CreateInstance(Host);
trainer = inst as TScalarTrainer;
if (trainer == null)
{
var allTypes = TrainerHelper.GetParentTypes(inst.GetType()).ToArray();
var allTypesStr = string.Join("\n", allTypes.Select(c => c.ToString()));
throw ch.ExceptNotSupp($"Unable to cast {inst.GetType()} into {typeof(TScalarTrainer)}.\n-TYPES-\n{allTypesStr}");
}
}
_trainer = null;
_predictors[i] = TrainPredictor(ch, trainer, data, count);
}
}
return(CreatePredictor());
}
public ActionResult Index(long userId, UserLanguages userLanguages)
{
if (WebSettingsConfig.Instance.IsSectionForbidden(SectionId.MyKnowledge))
{
return(RedirectToAction("Index", RouteConfig.MAIN_CONTROLLER_NAME));
}
if (IsInvalid(userId, userLanguages))
{
return(RedirectToKnowledgeWall());
}
IUserRepetitionIntervalQuery userRepetitionIntervalQuery = CreateUserRepetitionIntervalQuery(userId,
RepetitionType.
All);
var trainerHelper = new TrainerHelper(userRepetitionIntervalQuery, userLanguages);
TrainerModel model = trainerHelper.GetTrainerModel(Request);
var pageRequiredData = new PageRequiredData(SectionId.MyKnowledge);
pageRequiredData.Title = WebSettingsConfig.Instance.GetTemplateText(pageRequiredData.SectionId, PageId.Cards, TemplateId.Title, PageName);
pageRequiredData.Keywords = WebSettingsConfig.Instance.GetTemplateText(pageRequiredData.SectionId, PageId.Cards, TemplateId.Keywords, PageName);
pageRequiredData.Description = WebSettingsConfig.Instance.GetTemplateText(pageRequiredData.SectionId, PageId.Cards, TemplateId.Description, PageName);
model.PageRequiredData = pageRequiredData;
model.SetMarkUrl = Url.Action("SetMark", ControllerName, null, Request.Url.Scheme);
model.BreadcrumbsItems = new List <BreadcrumbItem> {
new BreadcrumbItem
{
Title = "Стена знаний", Action = "Index", ControllerName = RouteConfig.USER_KNOWLEDGE_CONTROLLER
},
new BreadcrumbItem {
IsActive = true, Title = PageName
}
};
if (EnumerableValidator.IsNullOrEmpty(model.Items))
{
return(RedirectToKnowledgeWall());
}
ViewData[OurHtmlHelper.ViewDataFlags.SKIP_POPUP_PANEL] = true;
return(View("../Trainer/Index", model));
}
/// <summary>
/// Trains a random trainer.
/// </summary>
/// <param name="inputs">The inputs.</param>
/// <param name="predictWindow">The predict window.</param>
public static double RandomTrainerMethod(int inputs, int predictWindow)
{
double[] firstinput = MakeInputs(inputs);
double[] SecondInput = MakeInputs(inputs);
double[] ThirdInputs = MakeInputs(inputs);
double[] FourthInputs = MakeInputs(inputs);
double[] inp5 = MakeInputs(inputs);
double[] inp6 = MakeInputs(inputs);
var pair = TrainerHelper.ProcessPairs(firstinput, firstinput, inputs, predictWindow);
var pair2 = TrainerHelper.ProcessPairs(SecondInput, firstinput, inputs, predictWindow);
var pair3 = TrainerHelper.ProcessPairs(ThirdInputs, firstinput, inputs, predictWindow);
var pair4 = TrainerHelper.ProcessPairs(FourthInputs, firstinput, inputs, predictWindow);
var pair5 = TrainerHelper.ProcessPairs(inp5, firstinput, inputs, predictWindow);
var pair6 = TrainerHelper.ProcessPairs(inp6, firstinput, inputs, predictWindow);
BasicMLDataSet SuperSet = new BasicMLDataSet();
SuperSet.Add(pair);
SuperSet.Add(pair2);
SuperSet.Add(pair3);
SuperSet.Add(pair4);
var network = new BasicNetwork();
network.AddLayer(new BasicLayer(new ActivationTANH(), true, SuperSet.InputSize));
network.AddLayer(new BasicLayer(new ActivationTANH(), false, 20));
network.AddLayer(new BasicLayer(new ActivationTANH(), true, 0));
network.AddLayer(new BasicLayer(new ActivationLinear(), true, predictWindow));
//var layer = new BasicLayer(new ActivationTANH(), true, SuperSet.InputSize);
//layer.Network = network;
network.Structure.FinalizeStructure();
network.Reset();
// var network = (BasicNetwork)CreateEval.CreateElmanNetwork(SuperSet.InputSize, SuperSet.IdealSize);
return(CreateEval.TrainNetworks(network, SuperSet));
//Lets create an evaluation.
//Console.WriteLine(@"Last error rate on random trainer:" + error);
}
/// <summary>
/// Trains the pipeline with data coming from a <see cref="IDataView"/>.
/// </summary>
public ScikitPipeline Train(IDataView data,
string feature = "Feature", string label = null,
string weight = null, string groupId = null)
{
IDataView trans = data;
using (var ch = _env.Start("Create transforms"))
{
for (int i = 0; i < _transforms.Length; ++i)
{
try
{
trans = _env.CreateTransform(_transforms[i].transformSettings, trans);
}
catch (Exception e)
{
if (e.ToString().Contains("Unknown loadable class"))
{
var nn = _env.ComponentCatalog.GetAllClasses().Length;
var filt = _env.ComponentCatalog.GetAllClasses()
.Select(c => c.UserName)
.OrderBy(c => c)
.Where(c => c.Trim().Length > 2);
var regis = string.Join("\n", filt);
throw Contracts.Except(e, $"Unable to create transform '{_transforms[i].transformSettings}', assembly not registered among {nn}\n{regis}");
}
throw e;
}
_transforms[i].transform = trans as IDataTransform;
}
}
if (_predictor != null)
{
using (var ch = _env.Start("Create Predictor"))
{
_predictor.trainer = TrainerHelper.CreateTrainer(_env, _predictor.trainerSettings);
_roles = new List <KeyValuePair <RoleMappedSchema.ColumnRole, string> >();
_roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Feature, feature));
if (!string.IsNullOrEmpty(label))
{
_roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Label, label));
}
if (!string.IsNullOrEmpty(groupId))
{
_roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Group, groupId));
}
if (!string.IsNullOrEmpty(weight))
{
_roles.Add(new KeyValuePair <RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Weight, weight));
}
var roleMap = new RoleMappedData(trans, label, feature, group: groupId, weight: weight);
_predictor.predictor = _predictor.trainer.Train(_env, ch, roleMap);
_predictor.roleMapData = roleMap;
}
}
else
{
_predictor = new StepPredictor()
{
predictor = null,
trainer = null,
trainerSettings = null,
roleMapData = new RoleMappedData(trans)
};
// We predict one to make sure everything works fine.
using (var ch = _env.Start("Compute one prediction."))
{
var df = DataFrameIO.ReadView(trans, 1, keepVectors: true, env: _env);
if (df.Length == 0)
{
throw _env.ExceptEmpty("Something went wrong. The pipeline does not produce any output.");
}
}
}
return(this);
}
