public async Task SaveParameterAsync(ParameterStorage parameterStorage, int coa_id, int cat_id, double loss)
{
Console.WriteLine($"Parameter für {coa_id} und {cat_id} wird eingetragen");
using (SqlConnection sql = new SqlConnection(ki_write_output))
{
await sql.OpenAsync();
SqlCommand command = sql.CreateCommand();
string parameter = parameterStorage.GetParameterAsString();
//pfusch
if (loss > 10000) //should NEVER be that high! optimal error is <1
{
Console.WriteLine("Fehler viel zu hoch");
while (loss > 10000) //stackoverflow otherwise
{
loss = loss / 1000;
}
}
string error = loss.ToString().Replace(',', '.');
command.CommandText = $"INSERT INTO output_data (coa_id, cat_id, value, error) VALUES ({coa_id},{cat_id},'{parameter}',{error});";
Console.WriteLine(command.CommandText);
await command.ExecuteNonQueryAsync();
}
}
public void SaveModelAsParameter(Model modelContainer, int coa_id, int cat_id, double loss)
{
ITransformer model = modelContainer.trainedModel;
IEnumerable <ITransformer> chain = model as IEnumerable <ITransformer>;
ISingleFeaturePredictionTransformer <object> predictionTransformer =
chain.Last() as ISingleFeaturePredictionTransformer <object>;
object modelParameters = predictionTransformer.Model;
ParameterStorage ps = new ParameterStorage();
Console.WriteLine();
}
private async Task <List <CategoriesWithYearsAndValues> > GenerateAsync(string country, List <int> kategorienIDs, int from, int futureYear)
{
Countrystats countrystats = new Countrystats(); //Klasse für alle Kategorien und deren Werte per Jahr
countrystats.Country = new Country(country); //Land zu dem die Kategorien mit Werte gehören
countrystats.ListWithCategoriesWithYearsAndValues = await dB.GetCategoriesWithValuesAndYearsAsync(country, kategorienIDs); //Werte mit Jahren
categorycount = countrystats.ListWithCategoriesWithYearsAndValues.Count; //wie viele kategorien an daten für dieses land existieren
List <CategoriesWithYearsAndValues> CategorysWithFutureValues = new List <CategoriesWithYearsAndValues>();
Task <List <YearWithValue> >[] liste = new Task <List <YearWithValue> > [categorycount]; //liste damit jede kategorie in einem task abgearbeitet werden kann
List <YearWithValue> PopulationTotal = new List <YearWithValue>();
//Arbeite jede Kategorie parallel ab
for (int i = 0; i < categorycount; i++)
{
//Erstelle für jede Kategorie einen Liste mit eigenen Datensätzen
List <YearWithValue> SingleCategoryData = new List <YearWithValue>();
//Hole einzelne Datensätze für jedes Jahr heraus
foreach (var YearWithValue in countrystats.ListWithCategoriesWithYearsAndValues[i].YearsWithValues)
{
SingleCategoryData.Add(new YearWithValue(YearWithValue.Year, new Wert(Convert.ToDecimal(YearWithValue.Value.value)), countrystats.Country.name, YearWithValue.cat_id));
}
//Wenn ein Wert nicht dokumentiert ist, ist in der Datenbank 0 drin. Das verfälscht den Wert für die Ki
//entferne deswegen 0
SingleCategoryData = AI.RemoveZero(SingleCategoryData);
//Wenn es mindestens ein Jahr einer Kategorie gibt, in der der Wert nicht 0 ist
if (SingleCategoryData.Count > 1)
{
int coaid = await dB.GetCountryByNameAsync(country); //numeric of country
int categ = await dB.GetCategoryByNameAsync(countrystats.ListWithCategoriesWithYearsAndValues[i].category.name); //numeric of category
//Bearbeite eigenen Datensatz
int multi = Scale(SingleCategoryData) - 1; //wie viel man die normierten werte mulitplizieren muss damit sie wieder echt sind
if (SingleCategoryData.Any(x => x.cat_id == 4))
{
PopulationTotal = SingleCategoryData;
}
if (DifferentValuesCount(SingleCategoryData) > 2)
{
//linear train
liste[i] = Task.Run(async() =>
{
if (await dB.GetMaxYearAsync(coaid, categ) >= futureYear) //if all wanted values are already known
{
return(SingleCategoryData);
}
else
{
if ((SingleCategoryData.Any(x => x.cat_id > 38 && x.cat_id < 46)) || SingleCategoryData.Any(x => x.cat_id == 77)) //if categoy is an emission-type or temp
{
ML mL = new ML(dB);
if (dB.CheckModel(coaid, categ)) //check for model
{
List <YearWithValue> yearWithValues = new List <YearWithValue>();
if (categ == 77) //if temp
{
Model model = dB.LoadModel(0, 77);
yearWithValues = await mL.PredictTempOverYearsAsync(model, futureYear, SingleCategoryData, countrystats.Country);
return(yearWithValues);
}
else
{
Model modelContainer = dB.LoadModel(coaid, categ);
yearWithValues = await mL.PredictCo2OverYearsAsync(modelContainer, futureYear, coaid, SingleCategoryData, cNTK);
}
return(yearWithValues);
}
else //calculate model
{
if (categ == 77) //if temp
{
Model model = await mL.TrainTempModelAsync(countrystats.Country);
List <YearWithValue> x = await mL.PredictTempOverYearsAsync(model, futureYear, SingleCategoryData, countrystats.Country);
return(x);
}
else
{
List <YearWithValue> x = await mL.TrainAndPredictEmissionsAsync(SingleCategoryData, PopulationTotal, futureYear);
return(x);
}
}
}
else //if category is non-emission and no temp
{
bool parameterExists = await dB.CheckParametersAsync(coaid, categ); //check if parameter for this country and this category exist
if (parameterExists)
{
Console.WriteLine("Daten werden von Datenbank genommen");
ParameterStorage parStor = await dB.GetParameterAsync(coaid, categ); //Bekomme Parameter
List <YearWithValue> yearWithValues = new List <YearWithValue>();
foreach (var item in countrystats.ListWithCategoriesWithYearsAndValues[i - 1].YearsWithValues)
{
yearWithValues.Add(new YearWithValue(item.Year, new Wert(Convert.ToDecimal(item.Value.value)), countrystats.Country.name, item.cat_id));
}
yearWithValues = AI.RemoveZero(yearWithValues);
yearWithValues = cNTK.Predict(yearWithValues, from, futureYear, parStor);
return(yearWithValues);
}
else
{
CNTK cNTK = new CNTK(dB);
List <YearWithValue> x = await cNTK.TrainLinearOneOutputAsync(SingleCategoryData, futureYear);
if (SingleCategoryData.Any(a => a.cat_id == 4))
{
PopulationTotal = x;
}
return(PopulationTotal);
}
}
}
//
});
}
else
{
liste[i] = Task <List <YearWithValue> > .Run(() =>
{
return(sigmoid.TrainSigmoid(SingleCategoryData, futureYear, multi));
});
}
}
//ohne dieses else gäbe es einige leere Tasks im Array -> Exception
//ohne if geht die KI datensätze ohne einträge durch -> Verschwendung von Rechenleistung und Zeit
else
{
liste[i] = Task.Run(() => { return(new List <YearWithValue>()); });
}
}
//Warte parallel bis alle Kategorien gelernt und berechnet wurden
Task.WaitAll(liste);
//returne alle Kategorien
for (int i = 0; i < categorycount; i++)
{
CategorysWithFutureValues.Add(new CategoriesWithYearsAndValues(countrystats.ListWithCategoriesWithYearsAndValues[i].category, liste[i].Result));
}
return(CategorysWithFutureValues);
}
/// <summary>
/// Calculates future values based on alreadyknown Parameters
/// </summary>
/// <param name="yearWithValues">List that gets extended</param>
/// <param name="from">startyear</param>
/// <param name="futureYear">year in the future</param>
/// <param name="parStor">parameter</param>
/// <returns></returns>
public List <YearWithValue> Predict(List <YearWithValue> yearWithValues, int from, int futureYear, ParameterStorage parStor)
{
double j = yearWithValues.Max(k => k.Year);
float valueToDiv = CategoriesWithYearsAndValues.GetValuesFromList(yearWithValues).Max();
float[] inputs = CategoriesWithYearsAndValues.GetYearsFromList(yearWithValues);
List <double> listForTheNormalizedInputs = new List <double>();
foreach (var item in inputs)
{
listForTheNormalizedInputs.Add(item); //Small brain schleife?
}
Input input = StandardizationYears(listForTheNormalizedInputs, futureYear);
inputs = input.getAlleJahreNormiert();
if (j < futureYear)
{
float inputsMax = inputs.Max();
while (j < futureYear)
{
j++;
yearWithValues.Add(new YearWithValue(j, new Wert((parStor.W * inputsMax + parStor.b) * valueToDiv)));
float[] inputtemp = CategoriesWithYearsAndValues.GetYearsFromList(yearWithValues);
List <double> fuckinghelpme = new List <double>();
foreach (var item in inputtemp)
{
fuckinghelpme.Add(item); //Small brain schleife?
}
Input input2 = StandardizationYears(fuckinghelpme, futureYear);
inputtemp = input2.getAlleJahreNormiert();
inputsMax = inputtemp.Max();
}
}
else //cut list from year to futureyear
{
if (futureYear > from)
{
int indexMax = yearWithValues.FindIndex(a => a.Year == Convert.ToInt32(futureYear)); //finde Index von Jahr bis zu dem man Daten braucht
yearWithValues.RemoveRange(indexMax, yearWithValues.Count - indexMax); //Cutte List von Jahr bis zu dem man es braucht bis Ende
int indexMin = yearWithValues.FindIndex(b => b.Year == Convert.ToInt32(from));
yearWithValues.RemoveRange(0, indexMin);
}
else
{
var temp = yearWithValues.Where(x => x.Year == from);
yearWithValues = temp.ToList();;
}
}
return(yearWithValues);
}
public async Task <List <YearWithValue> > TrainLinearOneOutputAsync(List <YearWithValue> KnownValues, int FutureYear)
{
var device = DeviceDescriptor.UseDefaultDevice();
////Step 2: define values, and variables
Variable x = Variable.InputVariable(new NDShape(1, 1), DataType.Float, "input");
Variable y = Variable.InputVariable(new NDShape(1, 1), DataType.Float, "output");
////Step 2: define training data set from table above
float[] inputs = CategoriesWithYearsAndValues.GetYearsFromList(KnownValues);
List <double> temp = new List <double>();
foreach (var item in inputs)
{
temp.Add(item); //Small brain schleife?
}
Input input = StandardizationYears(temp, FutureYear);
inputs = input.getAlleJahreNormiert();
float[] outputs = CategoriesWithYearsAndValues.GetValuesFromList(KnownValues);
//Value.CreateBatch(Tensor(Achsen, Dimension), Werte, cpu/gpu)
float[] outputsnormiert = new float[outputs.Count()];
float WertZumDividieren = outputs.Max();
for (int i = 0; i < outputs.Length; i++)
{
outputsnormiert[i] = outputs[i] / WertZumDividieren;
}
//Werte normiert lassen, sonst stackoverflow :>
var xValues = Value.CreateBatch(new NDShape(1, 1), GetLastNValues(inputs, inputs.Length, input.step), device);
var yValues = Value.CreateBatch(new NDShape(1, 1), GetLastNValues(outputsnormiert, outputs.Length, input.step), device);
////Step 3: create linear regression model
var lr = createLRModel(x, device);
////Network model contains only two parameters b and w, so we query
////the model in order to get parameter values
var paramValues = lr.Inputs.Where(z => z.IsParameter).ToList();
var totalParameters = paramValues.Sum(c => c.Shape.TotalSize);
////Step 4: create trainer
var trainer = createTrainer(lr, y);
////Ştep 5: training
double b = 0, w = 0;
int max = 2000;
for (int i = 1; i <= max; i++)
{
var d = new Dictionary <Variable, Value>();
d.Add(x, xValues);
d.Add(y, yValues);
//
trainer.TrainMinibatch(d, true, device);
//
var loss = trainer.PreviousMinibatchLossAverage();
var eval = trainer.PreviousMinibatchEvaluationAverage();
//
if (i % 200 == 0)
{
Console.WriteLine($"It={i}, Loss={loss}, Eval={eval}");
}
if (i == max)
{
//print weights
var b0_name = paramValues[0].Name;
var b0 = new Value(paramValues[0].GetValue()).GetDenseData <float>(paramValues[0]);
var b1_name = paramValues[1].Name;
var b1 = new Value(paramValues[1].GetValue()).GetDenseData <float>(paramValues[1]);
Console.WriteLine($" ");
Console.WriteLine($"Training process finished with the following regression parameters:");
Console.WriteLine($"b={b0[0][0]}, w={b1[0][0]}");
b = b0[0][0];
w = b1[0][0];
ParameterStorage ps = new ParameterStorage(float.Parse(w.ToString()), float.Parse(b.ToString()));
int coaid = await dB.GetCountryByNameAsync(KnownValues.Where(k => k.Name != null).First().Name);
await dB.SaveParameterAsync(ps, coaid, KnownValues.Where(k => k.cat_id != 0).First().cat_id, loss);
Console.WriteLine(KnownValues.Min(k => k.Year));
KnownValues = Predict(KnownValues, Convert.ToInt32(KnownValues.Min(k => k.Year)), FutureYear, ps);
}
}
return(KnownValues);
}
