public async Task SaveParameterAsync(ParameterStorage parameterStorage, int coa_id, int cat_id, double loss) { Console.WriteLine($"Parameter für {coa_id} und {cat_id} wird eingetragen"); using (SqlConnection sql = new SqlConnection(ki_write_output)) { await sql.OpenAsync(); SqlCommand command = sql.CreateCommand(); string parameter = parameterStorage.GetParameterAsString(); //pfusch if (loss > 10000) //should NEVER be that high! optimal error is <1 { Console.WriteLine("Fehler viel zu hoch"); while (loss > 10000) //stackoverflow otherwise { loss = loss / 1000; } } string error = loss.ToString().Replace(',', '.'); command.CommandText = $"INSERT INTO output_data (coa_id, cat_id, value, error) VALUES ({coa_id},{cat_id},'{parameter}',{error});"; Console.WriteLine(command.CommandText); await command.ExecuteNonQueryAsync(); } }
public void SaveModelAsParameter(Model modelContainer, int coa_id, int cat_id, double loss) { ITransformer model = modelContainer.trainedModel; IEnumerable <ITransformer> chain = model as IEnumerable <ITransformer>; ISingleFeaturePredictionTransformer <object> predictionTransformer = chain.Last() as ISingleFeaturePredictionTransformer <object>; object modelParameters = predictionTransformer.Model; ParameterStorage ps = new ParameterStorage(); Console.WriteLine(); }
private async Task <List <CategoriesWithYearsAndValues> > GenerateAsync(string country, List <int> kategorienIDs, int from, int futureYear) { Countrystats countrystats = new Countrystats(); //Klasse für alle Kategorien und deren Werte per Jahr countrystats.Country = new Country(country); //Land zu dem die Kategorien mit Werte gehören countrystats.ListWithCategoriesWithYearsAndValues = await dB.GetCategoriesWithValuesAndYearsAsync(country, kategorienIDs); //Werte mit Jahren categorycount = countrystats.ListWithCategoriesWithYearsAndValues.Count; //wie viele kategorien an daten für dieses land existieren List <CategoriesWithYearsAndValues> CategorysWithFutureValues = new List <CategoriesWithYearsAndValues>(); Task <List <YearWithValue> >[] liste = new Task <List <YearWithValue> > [categorycount]; //liste damit jede kategorie in einem task abgearbeitet werden kann List <YearWithValue> PopulationTotal = new List <YearWithValue>(); //Arbeite jede Kategorie parallel ab for (int i = 0; i < categorycount; i++) { //Erstelle für jede Kategorie einen Liste mit eigenen Datensätzen List <YearWithValue> SingleCategoryData = new List <YearWithValue>(); //Hole einzelne Datensätze für jedes Jahr heraus foreach (var YearWithValue in countrystats.ListWithCategoriesWithYearsAndValues[i].YearsWithValues) { SingleCategoryData.Add(new YearWithValue(YearWithValue.Year, new Wert(Convert.ToDecimal(YearWithValue.Value.value)), countrystats.Country.name, YearWithValue.cat_id)); } //Wenn ein Wert nicht dokumentiert ist, ist in der Datenbank 0 drin. Das verfälscht den Wert für die Ki //entferne deswegen 0 SingleCategoryData = AI.RemoveZero(SingleCategoryData); //Wenn es mindestens ein Jahr einer Kategorie gibt, in der der Wert nicht 0 ist if (SingleCategoryData.Count > 1) { int coaid = await dB.GetCountryByNameAsync(country); //numeric of country int categ = await dB.GetCategoryByNameAsync(countrystats.ListWithCategoriesWithYearsAndValues[i].category.name); //numeric of category //Bearbeite eigenen Datensatz int multi = Scale(SingleCategoryData) - 1; //wie viel man die normierten werte mulitplizieren muss damit sie wieder echt sind if (SingleCategoryData.Any(x => x.cat_id == 4)) { PopulationTotal = SingleCategoryData; } if (DifferentValuesCount(SingleCategoryData) > 2) { //linear train liste[i] = Task.Run(async() => { if (await dB.GetMaxYearAsync(coaid, categ) >= futureYear) //if all wanted values are already known { return(SingleCategoryData); } else { if ((SingleCategoryData.Any(x => x.cat_id > 38 && x.cat_id < 46)) || SingleCategoryData.Any(x => x.cat_id == 77)) //if categoy is an emission-type or temp { ML mL = new ML(dB); if (dB.CheckModel(coaid, categ)) //check for model { List <YearWithValue> yearWithValues = new List <YearWithValue>(); if (categ == 77) //if temp { Model model = dB.LoadModel(0, 77); yearWithValues = await mL.PredictTempOverYearsAsync(model, futureYear, SingleCategoryData, countrystats.Country); return(yearWithValues); } else { Model modelContainer = dB.LoadModel(coaid, categ); yearWithValues = await mL.PredictCo2OverYearsAsync(modelContainer, futureYear, coaid, SingleCategoryData, cNTK); } return(yearWithValues); } else //calculate model { if (categ == 77) //if temp { Model model = await mL.TrainTempModelAsync(countrystats.Country); List <YearWithValue> x = await mL.PredictTempOverYearsAsync(model, futureYear, SingleCategoryData, countrystats.Country); return(x); } else { List <YearWithValue> x = await mL.TrainAndPredictEmissionsAsync(SingleCategoryData, PopulationTotal, futureYear); return(x); } } } else //if category is non-emission and no temp { bool parameterExists = await dB.CheckParametersAsync(coaid, categ); //check if parameter for this country and this category exist if (parameterExists) { Console.WriteLine("Daten werden von Datenbank genommen"); ParameterStorage parStor = await dB.GetParameterAsync(coaid, categ); //Bekomme Parameter List <YearWithValue> yearWithValues = new List <YearWithValue>(); foreach (var item in countrystats.ListWithCategoriesWithYearsAndValues[i - 1].YearsWithValues) { yearWithValues.Add(new YearWithValue(item.Year, new Wert(Convert.ToDecimal(item.Value.value)), countrystats.Country.name, item.cat_id)); } yearWithValues = AI.RemoveZero(yearWithValues); yearWithValues = cNTK.Predict(yearWithValues, from, futureYear, parStor); return(yearWithValues); } else { CNTK cNTK = new CNTK(dB); List <YearWithValue> x = await cNTK.TrainLinearOneOutputAsync(SingleCategoryData, futureYear); if (SingleCategoryData.Any(a => a.cat_id == 4)) { PopulationTotal = x; } return(PopulationTotal); } } } // }); } else { liste[i] = Task <List <YearWithValue> > .Run(() => { return(sigmoid.TrainSigmoid(SingleCategoryData, futureYear, multi)); }); } } //ohne dieses else gäbe es einige leere Tasks im Array -> Exception //ohne if geht die KI datensätze ohne einträge durch -> Verschwendung von Rechenleistung und Zeit else { liste[i] = Task.Run(() => { return(new List <YearWithValue>()); }); } } //Warte parallel bis alle Kategorien gelernt und berechnet wurden Task.WaitAll(liste); //returne alle Kategorien for (int i = 0; i < categorycount; i++) { CategorysWithFutureValues.Add(new CategoriesWithYearsAndValues(countrystats.ListWithCategoriesWithYearsAndValues[i].category, liste[i].Result)); } return(CategorysWithFutureValues); }
/// <summary> /// Calculates future values based on alreadyknown Parameters /// </summary> /// <param name="yearWithValues">List that gets extended</param> /// <param name="from">startyear</param> /// <param name="futureYear">year in the future</param> /// <param name="parStor">parameter</param> /// <returns></returns> public List <YearWithValue> Predict(List <YearWithValue> yearWithValues, int from, int futureYear, ParameterStorage parStor) { double j = yearWithValues.Max(k => k.Year); float valueToDiv = CategoriesWithYearsAndValues.GetValuesFromList(yearWithValues).Max(); float[] inputs = CategoriesWithYearsAndValues.GetYearsFromList(yearWithValues); List <double> listForTheNormalizedInputs = new List <double>(); foreach (var item in inputs) { listForTheNormalizedInputs.Add(item); //Small brain schleife? } Input input = StandardizationYears(listForTheNormalizedInputs, futureYear); inputs = input.getAlleJahreNormiert(); if (j < futureYear) { float inputsMax = inputs.Max(); while (j < futureYear) { j++; yearWithValues.Add(new YearWithValue(j, new Wert((parStor.W * inputsMax + parStor.b) * valueToDiv))); float[] inputtemp = CategoriesWithYearsAndValues.GetYearsFromList(yearWithValues); List <double> fuckinghelpme = new List <double>(); foreach (var item in inputtemp) { fuckinghelpme.Add(item); //Small brain schleife? } Input input2 = StandardizationYears(fuckinghelpme, futureYear); inputtemp = input2.getAlleJahreNormiert(); inputsMax = inputtemp.Max(); } } else //cut list from year to futureyear { if (futureYear > from) { int indexMax = yearWithValues.FindIndex(a => a.Year == Convert.ToInt32(futureYear)); //finde Index von Jahr bis zu dem man Daten braucht yearWithValues.RemoveRange(indexMax, yearWithValues.Count - indexMax); //Cutte List von Jahr bis zu dem man es braucht bis Ende int indexMin = yearWithValues.FindIndex(b => b.Year == Convert.ToInt32(from)); yearWithValues.RemoveRange(0, indexMin); } else { var temp = yearWithValues.Where(x => x.Year == from); yearWithValues = temp.ToList();; } } return(yearWithValues); }
public async Task <List <YearWithValue> > TrainLinearOneOutputAsync(List <YearWithValue> KnownValues, int FutureYear) { var device = DeviceDescriptor.UseDefaultDevice(); ////Step 2: define values, and variables Variable x = Variable.InputVariable(new NDShape(1, 1), DataType.Float, "input"); Variable y = Variable.InputVariable(new NDShape(1, 1), DataType.Float, "output"); ////Step 2: define training data set from table above float[] inputs = CategoriesWithYearsAndValues.GetYearsFromList(KnownValues); List <double> temp = new List <double>(); foreach (var item in inputs) { temp.Add(item); //Small brain schleife? } Input input = StandardizationYears(temp, FutureYear); inputs = input.getAlleJahreNormiert(); float[] outputs = CategoriesWithYearsAndValues.GetValuesFromList(KnownValues); //Value.CreateBatch(Tensor(Achsen, Dimension), Werte, cpu/gpu) float[] outputsnormiert = new float[outputs.Count()]; float WertZumDividieren = outputs.Max(); for (int i = 0; i < outputs.Length; i++) { outputsnormiert[i] = outputs[i] / WertZumDividieren; } //Werte normiert lassen, sonst stackoverflow :> var xValues = Value.CreateBatch(new NDShape(1, 1), GetLastNValues(inputs, inputs.Length, input.step), device); var yValues = Value.CreateBatch(new NDShape(1, 1), GetLastNValues(outputsnormiert, outputs.Length, input.step), device); ////Step 3: create linear regression model var lr = createLRModel(x, device); ////Network model contains only two parameters b and w, so we query ////the model in order to get parameter values var paramValues = lr.Inputs.Where(z => z.IsParameter).ToList(); var totalParameters = paramValues.Sum(c => c.Shape.TotalSize); ////Step 4: create trainer var trainer = createTrainer(lr, y); ////Ştep 5: training double b = 0, w = 0; int max = 2000; for (int i = 1; i <= max; i++) { var d = new Dictionary <Variable, Value>(); d.Add(x, xValues); d.Add(y, yValues); // trainer.TrainMinibatch(d, true, device); // var loss = trainer.PreviousMinibatchLossAverage(); var eval = trainer.PreviousMinibatchEvaluationAverage(); // if (i % 200 == 0) { Console.WriteLine($"It={i}, Loss={loss}, Eval={eval}"); } if (i == max) { //print weights var b0_name = paramValues[0].Name; var b0 = new Value(paramValues[0].GetValue()).GetDenseData <float>(paramValues[0]); var b1_name = paramValues[1].Name; var b1 = new Value(paramValues[1].GetValue()).GetDenseData <float>(paramValues[1]); Console.WriteLine($" "); Console.WriteLine($"Training process finished with the following regression parameters:"); Console.WriteLine($"b={b0[0][0]}, w={b1[0][0]}"); b = b0[0][0]; w = b1[0][0]; ParameterStorage ps = new ParameterStorage(float.Parse(w.ToString()), float.Parse(b.ToString())); int coaid = await dB.GetCountryByNameAsync(KnownValues.Where(k => k.Name != null).First().Name); await dB.SaveParameterAsync(ps, coaid, KnownValues.Where(k => k.cat_id != 0).First().cat_id, loss); Console.WriteLine(KnownValues.Min(k => k.Year)); KnownValues = Predict(KnownValues, Convert.ToInt32(KnownValues.Min(k => k.Year)), FutureYear, ps); } } return(KnownValues); }