private PredictorEntity CrossOver(PredictorEntity child, PredictorEntity father, PredictorEntity mother, Random r, AutoconfigureNeuralNetworkEntity conf)
{
var nnChild = (NeuralNetworkSettingsEntity)child.AlgorithmSettings;
var nnFather = (NeuralNetworkSettingsEntity)father.AlgorithmSettings;
var nnMother = (NeuralNetworkSettingsEntity)mother.AlgorithmSettings;
if (conf.ExploreLearner)
{
nnChild.Learner = r.NextBool() ? nnFather.Learner : nnMother.Learner;
}
if (conf.ExploreLearningValues)
{
nnChild.LearningRate = r.NextBool() ? nnFather.LearningRate : nnMother.LearningRate;
nnChild.LearningMomentum = r.NextBool() ? nnFather.LearningMomentum : nnMother.LearningMomentum;
nnChild.LearningVarianceMomentum = r.NextBool() ? nnFather.LearningVarianceMomentum : nnMother.LearningVarianceMomentum;
}
if (conf.ExploreHiddenLayers)
{
nnChild.HiddenLayers = nnFather.HiddenLayers.ZipOrDefault(nnMother.HiddenLayers, (h1, h2) => r.NextBool() ? h1 : h2).NotNull().ToMList();
}
if (conf.ExploreOutputLayer)
{
nnChild.OutputActivation = r.NextBool() ? nnFather.OutputActivation : nnMother.OutputActivation;
nnChild.OutputInitializer = r.NextBool() ? nnFather.OutputInitializer : nnMother.OutputInitializer;
}
return(child);
}
static void StartTrainingAsync(PredictorEntity p)
{
var cancellationSource = new CancellationTokenSource();
var ctx = new PredictorTrainingContext(p, cancellationSource.Token);
var state = new PredictorTrainingState(cancellationSource, ctx);
if (!Trainings.TryAdd(p.ToLite(), state))
{
throw new InvalidOperationException(PredictorMessage._0IsAlreadyBeingTrained.NiceToString(p));
}
using (ExecutionContext.SuppressFlow())
{
Task.Run(() =>
{
var user = ExecutionMode.Global().Using(_ => p.User !.RetrieveAndRemember());
using (UserHolder.UserSession(user))
{
try
{
DoTraining(ctx);
}
finally
{
Trainings.TryRemove(p.ToLite(), out var _);
}
}
});
}
}
static void PredictorEntity_Retrieved(PredictorEntity predictor)
{
object queryName = QueryLogic.ToQueryName(predictor.Query.Key);
QueryDescription description = DynamicQueryManager.Current.QueryDescription(queryName);
predictor.ParseData(description);
}
public static PredictorPredictContext CreatePredictContext(this PredictorEntity p)
{
var codifications = p.RetrievePredictorCodifications();
var ppc = new PredictorPredictContext(p, PredictorLogic.Algorithms.GetOrThrow(p.Algorithm), codifications);
ppc.Algorithm.LoadModel(ppc);
return(ppc);
}
public string?ValidateEncodingProperty(PredictorEntity predictor, PredictorSubQueryEntity?subQuery, PredictorColumnEncodingSymbol encoding, PredictorColumnUsage usage, QueryTokenEmbedded token)
{
if (!ReflectionTools.IsDecimalNumber(token.Token.Type) && !ReflectionTools.IsNumber(token.Token.Type))
{
return(PredictorMessage._0NotSuportedFor1.NiceToString(encoding.NiceToString(), predictor.Algorithm.NiceToString()));
}
return(null);
}
public PredictorColumnEmbedded AssertOnlyOutput(PredictorEntity predictor)
{
var outputs = predictor.MainQuery.Columns.Where(a => a.Usage == PredictorColumnUsage.Output);
if (outputs.Count() != 1)
{
throw new InvalidOperationException($"{predictor.ResultSaver} requires the predictor to have only one output (instead of {outputs.Count()})");
}
return(outputs.SingleEx());
}
public PredictorEpochProgressEntity SaveEntity(PredictorEntity predictor)
{
return(new PredictorEpochProgressEntity
{
Predictor = predictor.ToLite(),
Ellapsed = Ellapsed,
Epoch = Epoch,
TrainingExamples = TrainingExamples,
LossTraining = LossTraining?.CleanDouble(),
EvaluationTraining = EvaluationTraining?.CleanDouble(),
LossValidation = LossValidation?.CleanDouble(),
EvaluationValidation = EvaluationValidation?.CleanDouble(),
}.Save());
}
public string?ValidateEncodingProperty(PredictorEntity predictor, PredictorSubQueryEntity?subQuery, PredictorColumnEncodingSymbol encoding, PredictorColumnUsage usage, QueryTokenEmbedded token)
{
var nn = (NeuralNetworkSettingsEntity)predictor.AlgorithmSettings;
if (ReflectionTools.IsDecimalNumber(token.Token.Type))
{
return(PredictorMessage._0NotSuportedFor1.NiceToString(encoding.NiceToString(), predictor.Algorithm.NiceToString()));
}
if (usage == PredictorColumnUsage.Output && (nn.PredictionType == PredictionType.Regression || nn.PredictionType == PredictionType.MultiRegression))
{
return(PredictorMessage._0NotSuportedFor1.NiceToString(encoding.NiceToString(), nn.PredictionType.NiceToString()));
}
return(null);
}
static void CleanTrained(PredictorEntity e)
{
PredictorPredictLogic.TrainedPredictorCache.Remove(e.ToLite());
e.TrainingException = null;
foreach (var fp in e.Files)
{
fp.DeleteFileOnCommit();
}
e.ClassificationTraining = null;
e.ClassificationValidation = null;
e.RegressionTraining = null;
e.RegressionValidation = null;
e.Files.Clear();
e.Codifications().UnsafeDelete();
e.EpochProgresses().UnsafeDelete();
}
public string?ValidateEncodingProperty(PredictorEntity predictor, PredictorSubQueryEntity?subQuery, PredictorColumnEncodingSymbol encoding, PredictorColumnUsage usage, QueryTokenEmbedded token)
{
var nn = (NeuralNetworkSettingsEntity)predictor.AlgorithmSettings;
if (!ReflectionTools.IsNumber(token.Token.Type) && token.Token.Type.UnNullify() != typeof(bool))
{
return(PredictorMessage._0IsRequiredFor1.NiceToString(DefaultColumnEncodings.OneHot.NiceToString(), token.Token.NiceTypeName));
}
if (usage == PredictorColumnUsage.Output && (nn.PredictionType == PredictionType.Classification || nn.PredictionType == PredictionType.MultiClassification))
{
return(PredictorMessage._0NotSuportedFor1.NiceToString(encoding.NiceToString(), nn.PredictionType.NiceToString()));
}
return(null);
}
public string?ValidateEncodingProperty(PredictorEntity predictor, PredictorSubQueryEntity?subQuery, PredictorColumnEncodingSymbol encoding, PredictorColumnUsage usage, QueryTokenEmbedded token)
{
var nn = (NeuralNetworkSettingsEntity)predictor.AlgorithmSettings;
if (token.Token.Type != typeof(string))
{
return(PredictorMessage._0NotSuportedFor1.NiceToString(encoding.NiceToString(), predictor.Algorithm.NiceToString()));
}
if (usage == PredictorColumnUsage.Output)
{
return(PredictorMessage._0NotSuportedFor1.NiceToString(encoding.NiceToString(), usage.NiceToString()));
}
return(null);
}
public static QueryRequest ToQueryRequest(this PredictorEntity predictor)
{
return(new QueryRequest
{
QueryName = predictor.Query.ToQueryName(),
Filters = predictor.Filters.Select(f => ToFilter(f)).ToList(),
Columns = predictor.Columns
.Where(c => c.Type == PredictorColumnType.SimpleColumn)
.Select(c => new Column(c.Token.Token, null)).ToList(),
Pagination = new Pagination.All(),
Orders = Enumerable.Empty <Order>().ToList(),
});
}
public static void TrainSync(this PredictorEntity p, bool autoReset = true, Action <string, decimal?>?onReportProgres = null, CancellationToken?cancellationToken = null)
{
if (autoReset)
{
if (p.State == PredictorState.Trained || p.State == PredictorState.Error)
{
p.Execute(PredictorOperation.Untrain);
}
else if (p.State == PredictorState.Training)
{
p.Execute(PredictorOperation.CancelTraining);
}
}
p.User = UserHolder.Current.ToLite();
p.State = PredictorState.Training;
p.Save();
var ctx = new PredictorTrainingContext(p, cancellationToken ?? new CancellationTokenSource().Token);
var lastWithProgress = false;
if (onReportProgres != null)
{
ctx.OnReportProgres += onReportProgres;
}
else
{
ctx.OnReportProgres += (message, progress) =>
{
if (progress == null)
{
if (lastWithProgress)
{
Console.WriteLine();
}
SafeConsole.WriteLineColor(ConsoleColor.White, message);
}
else
{
SafeConsole.WriteLineColor(ConsoleColor.White, $"{progress:P} - {message}");
lastWithProgress = true;
}
}
};
DoTraining(ctx);
}
public PredictorPredictContext(PredictorEntity predictor, IPredictorAlgorithm algorithm, List <PredictorCodification> codifications)
{
Predictor = predictor;
Algorithm = algorithm;
Codifications = codifications;
InputCodifications = codifications.Where(a => a.Column.Usage == PredictorColumnUsage.Input).ToList();
InputCodificationsByColumn = InputCodifications.GroupToDictionary(a => a.Column);
OutputCodifications = codifications.Where(a => a.Column.Usage == PredictorColumnUsage.Output).ToList();
MainOutputCodifications = OutputCodifications.Where(a => a.Column is PredictorColumnMain m).GroupToDictionary(a => ((PredictorColumnMain)a.Column).PredictorColumn);
SubQueryOutputCodifications = OutputCodifications.Where(a => a.Column is PredictorColumnSubQuery).AgGroupToDictionary(a => ((PredictorColumnSubQuery)a.Column).SubQuery, sqGroup =>
new PredictorPredictSubQueryContext(sqGroup.Key,
sqGroup.AgGroupToDictionary(a => ((PredictorColumnSubQuery)a.Column).Keys !,
keysGroup => keysGroup.GroupToDictionary(a => ((PredictorColumnSubQuery)a.Column).PredictorSubQueryColumn !), ObjectArrayComparer.Instance)
));
}
public static QueryGroupRequest ToMultiColumnQuery(this PredictorEntity predictor, Implementations mainQueryImplementations, PredictorMultiColumnEntity mc)
{
var mainQueryKey = mc.GroupKeys.FirstEx();
if (!Compatible(mainQueryKey.Token.GetImplementations(), mainQueryImplementations))
{
throw new InvalidOperationException($"{mainQueryKey.Token} of {mc.Query} should be of type {mainQueryImplementations}");
}
var mainFilters = predictor.Filters.Select(f => predictor.Query.Is(mc.Query) ? ToFilter(f) : ToFilterAppend(f, mainQueryKey.Token));
var additionalFilters = mc.AdditionalFilters.Select(f => ToFilter(f)).ToList();
var groupKeys = mc.GroupKeys.Select(c => new Column(c.Token, null)).ToList();
var aggregates = mc.Aggregates.Select(c => new Column(c.Token, null)).ToList();
return(new QueryGroupRequest
{
QueryName = mc.Query.ToQueryName(),
Filters = mainFilters.Concat(additionalFilters).ToList(),
Columns = groupKeys.Concat(aggregates).ToList(),
Orders = new List <Order>()
});
}
static void PredictorEntity_Retrieved(PredictorEntity predictor)
{
predictor.MainQuery.ParseData();
}
public void AssertValid(PredictorEntity predictor)
{
AssertOnlyOutput(predictor);
}
public static void ImportPredictor()
{
using (AuthLogic.UnsafeUserSession("System"))
{
var predictor = new PredictorEntity
{
Name = "Product Estimation",
Algorithm = TensorFlowPredictorAlgorithm.NeuralNetworkGraph,
ResultSaver = PredictorSimpleResultSaver.Full,
MainQuery = new PredictorMainQueryEmbedded
{
Query = QueryLogic.GetQueryEntity(typeof(OrderEntity)),
GroupResults = true,
Columns =
{
new PredictorColumnEmbedded
{
Usage = PredictorColumnUsage.Input,
Token = new QueryTokenEmbedded("Entity.OrderDate.Year"),
Encoding = DefaultColumnEncodings.NormalizeZScore
},
new PredictorColumnEmbedded
{
Usage = PredictorColumnUsage.Input,
Token = new QueryTokenEmbedded("Entity.OrderDate.Month"),
Encoding = DefaultColumnEncodings.NormalizeZScore
},
new PredictorColumnEmbedded
{
Usage = PredictorColumnUsage.Input,
Token = new QueryTokenEmbedded("Entity.Details.Element.Product"),
Encoding = DefaultColumnEncodings.OneHot
},
new PredictorColumnEmbedded
{
Usage = PredictorColumnUsage.Output,
Token = new QueryTokenEmbedded("Entity.Details.Element.Quantity.Sum"),
Encoding = DefaultColumnEncodings.NormalizeZScore
},
}
},
Settings = new PredictorSettingsEmbedded
{
TestPercentage = 0.05,
},
AlgorithmSettings = new NeuralNetworkSettingsEntity
{
PredictionType = PredictionType.Regression,
Optimizer = TensorFlowOptimizer.GradientDescentOptimizer,
LossFunction = NeuralNetworkEvalFunction.MeanSquaredError,
EvalErrorFunction = NeuralNetworkEvalFunction.MeanAbsoluteError,
LearningRate = 0.0001,
MinibatchSize = 100,
NumMinibatches = 1000,
}
}.ParseData().Execute(PredictorOperation.Save);
predictor.TrainSync();
predictor.Execute(PredictorOperation.Publish, ProductPredictorPublication.MonthlySales);
}
}//ImportPredictor
public PredictDictionary(PredictorEntity predictor, PredictionOptions?options = null, Lite <Entity>?entity = null)
{
Predictor = predictor;
Options = options;
Entity = entity;
}
private static double Evaluate(ExecutingProcess ep, PredictorEntity p, Action <decimal?> onProgress)
{
PredictorLogic.TrainSync(p, onReportProgres: (str, val) => onProgress(val));
return(p.ResultValidation !.Loss.Value);
}
public static IQueryable <PredictorCodificationEntity> Codifications(this PredictorEntity e) => As.Expression(() => Database.Query <PredictorCodificationEntity>().Where(a => a.Predictor.Is(e)));
static void PredictorEntity_Retrieved(PredictorEntity predictor, PostRetrievingContext ctx)
{
predictor.MainQuery.ParseData();
}
public static PredictorEntity ParseData(this PredictorEntity predictor)
{
predictor.MainQuery.ParseData();
predictor.SubQueries.ForEach(sq => sq.ParseData());
return(predictor);
}
public static List <PredictorCodification> RetrievePredictorCodifications(this PredictorEntity predictor)
{
var list = predictor.Codifications().ToList();
object?ParseValue(string str, QueryToken token)
{
return(FilterValueConverter.Parse(str, token.Type, isList: false));
}
object?ParseKey(string?str, PredictorSubQueryColumnEmbedded key)
{
return(FilterValueConverter.Parse(str, key.Token.Token.Type, isList: false));
}
object?[] GetKeys(PredictorCodificationEntity cod, List <PredictorSubQueryColumnEmbedded> keys)
{
switch (keys.Count)
{
case 0: return(new object[0]);
case 1:
return(new[] {
ParseKey(cod.SplitKey0, keys[0]),
});
case 2:
return(new[] {
ParseKey(cod.SplitKey0, keys[0]),
ParseKey(cod.SplitKey1, keys[1]),
});
case 3:
return(new[] {
ParseKey(cod.SplitKey0, keys[0]),
ParseKey(cod.SplitKey1, keys[1]),
ParseKey(cod.SplitKey2, keys[2]),
});
default:
throw new InvalidOperationException("Unexpected Group count");
}
}
Dictionary <int, PredictorColumnMain> mainColumns = new Dictionary <int, PredictorColumnMain>();
PredictorColumnMain GetPredictorColumnMain(PredictorCodificationEntity cod)
{
return(mainColumns.GetOrCreate(cod.OriginalColumnIndex, () => new PredictorColumnMain(
predictorColumnIndex: cod.OriginalColumnIndex,
predictorColumn: predictor.MainQuery.Columns[cod.OriginalColumnIndex]
)));
}
Dictionary <int, Dictionary <int, PredictorColumnSubQuery> > subColumns = new Dictionary <int, Dictionary <int, PredictorColumnSubQuery> >();
PredictorColumnSubQuery GetPredictorColumnSubQuery(PredictorCodificationEntity cod)
{
return(subColumns.GetOrCreate(cod.SubQueryIndex !.Value).GetOrCreate(cod.OriginalColumnIndex, () => {
var sq = predictor.SubQueries[cod.SubQueryIndex.Value];
var col = sq.Columns[cod.OriginalColumnIndex];
if (col.Usage == PredictorSubQueryColumnUsage.SplitBy || col.Usage == PredictorSubQueryColumnUsage.ParentKey)
{
throw new InvalidOperationException("Unexpected codification usage");
}
var keys = GetKeys(cod, sq.Columns.Where(a => a.Usage == PredictorSubQueryColumnUsage.SplitBy).ToList());
return new PredictorColumnSubQuery(col, cod.OriginalColumnIndex, sq, keys);
}));
}
return((from cod in list
let col = cod.SubQueryIndex == null ?
(PredictorColumnBase)GetPredictorColumnMain(cod) :
(PredictorColumnBase)GetPredictorColumnSubQuery(cod)
select new PredictorCodification(col)
{
Index = cod.Index,
IsValue = cod.IsValue != null ? ParseValue(cod.IsValue, col.Token) : null,
Average = cod.Average,
StdDev = cod.StdDev,
Min = cod.Min,
Max = cod.Max,
})
.ToList());
}
public string?ValidateEncodingProperty(PredictorEntity predictor, PredictorSubQueryEntity?subQuery, PredictorColumnEncodingSymbol encoding, PredictorColumnUsage usage, QueryTokenEmbedded token)
{
return(Encodings.GetOrThrow(encoding).ValidateEncodingProperty(predictor, subQuery, encoding, usage, token));
}
//private static double EvaluateMock(ExecutingProcess ep, PredictorEntity p, Action<decimal?> onProgress)
//{
// var nns = (NeuralNetworkSettingsEntity)p.AlgorithmSettings;
// var ctx = Lite.Create<PredictorEntity>(1835).GetPredictContext();
// var mq = ctx.Predictor.MainQuery;
// var inputs = new PredictDictionary(ctx.Predictor)
// {
// MainQueryValues =
// {
// { mq.FindColumn(nameof(nns.Learner)), nns.Learner },
// { mq.FindColumn(nameof(nns.LearningRate)), nns.LearningRate },
// { mq.FindColumn(nameof(nns.LearningMomentum)), nns.LearningMomentum },
// { mq.FindColumn(nameof(nns.LearningVarianceMomentum)), nns.LearningVarianceMomentum },
// { mq.FindColumn(nameof(nns.LearningUnitGain)), nns.LearningUnitGain },
// }
// };
// var outputs = inputs.PredictBasic();
// var outValue = outputs.MainQueryValues.GetOrThrow(mq.FindColumn(nameof(ctx.Predictor.ResultValidation)));
// return Convert.ToDouble(outValue);
//}
private void Mutate(PredictorEntity predictor, AutoconfigureNeuralNetworkEntity conf, double mutationProbability, Random r)
{
var nns = (NeuralNetworkSettingsEntity)predictor.AlgorithmSettings;
if (conf.ExploreLearner)
{
if (r.NextDouble() < mutationProbability)
{
nns.Learner = r.NextElement(EnumExtensions.GetValues <NeuralNetworkLearner>());
}
}
if (conf.ExploreLearningValues)
{
double IncrementOrDecrement(double value)
{
var ratio = 1.1 + r.NextDouble() * 0.9;
return(r.NextBool() ? value / ratio : value *ratio);
}
if (r.NextDouble() < mutationProbability)
{
nns.LearningRate = IncrementOrDecrement(nns.LearningRate);
}
if (r.NextDouble() < mutationProbability)
{
nns.LearningMomentum = IncrementOrDecrement(nns.LearningMomentum ?? 0.01);
}
if (r.Next() < mutationProbability)
{
nns.LearningVarianceMomentum = IncrementOrDecrement(nns.LearningVarianceMomentum ?? 0.01);
}
}
if (conf.ExploreHiddenLayers)
{
if (r.NextDouble() < mutationProbability)
{
var shouldHidden = Math.Min(0, Math.Max(nns.HiddenLayers.Count + (r.NextBool() ? 1 : -1), conf.MaxLayers));
if (shouldHidden > nns.HiddenLayers.Count)
{
nns.HiddenLayers.Add(new NeuralNetworkHidenLayerEmbedded
{
Size = r.Next(conf.MaxNeuronsPerLayer),
Activation = r.NextElement(EnumExtensions.GetValues <NeuralNetworkActivation>()),
Initializer = r.NextElement(EnumExtensions.GetValues <NeuralNetworkInitializer>()),
});
}
else if (shouldHidden < nns.HiddenLayers.Count)
{
nns.HiddenLayers.RemoveAt(r.Next(nns.HiddenLayers.Count));
}
}
foreach (var hl in nns.HiddenLayers)
{
if (r.NextDouble() < mutationProbability)
{
hl.Size = (r.Next(conf.MinNeuronsPerLayer, conf.MaxNeuronsPerLayer) + hl.Size) / 2;
}
if (r.NextDouble() < mutationProbability)
{
hl.Activation = r.NextElement(EnumExtensions.GetValues <NeuralNetworkActivation>());
}
if (r.NextDouble() < mutationProbability)
{
hl.Initializer = r.NextElement(EnumExtensions.GetValues <NeuralNetworkInitializer>());
}
}
}
if (conf.ExploreOutputLayer)
{
if (r.NextDouble() < mutationProbability)
{
nns.OutputActivation = r.NextElement(EnumExtensions.GetValues <NeuralNetworkActivation>());
}
if (r.NextDouble() < mutationProbability)
{
nns.OutputInitializer = r.NextElement(EnumExtensions.GetValues <NeuralNetworkInitializer>());
}
}
nns.LearningUnitGain = false; //better to deverge than to stay flat
}
public PredictorTrainingContext(PredictorEntity predictor, CancellationToken cancellationToken)
{
this.Predictor = predictor;
this.CancellationToken = cancellationToken;
}
public List <PredictorEntity> CrossOverPopulation(Dictionary <PredictorEntity, double> evaluatedPopulation, PredictorEntity initial, AutoconfigureNeuralNetworkEntity opt, Random r)
{
var positiveSurvivors = evaluatedPopulation.ToDictionary(kvp => kvp.Key, kvp => 1 / (kvp.Value + 0.01));
var total = positiveSurvivors.Values.Sum();
PredictorEntity SelectRandomly()
{
var point = r.NextDouble() * total;
double acum = 0;
foreach (var kvp in positiveSurvivors !)
{
acum += kvp.Value;
if (point < acum)
{
return(kvp.Key);
}
}
throw new InvalidOperationException("Out of range");
}
return(0.To(opt.Population).Select(i => CrossOver(initial.ConstructFrom(PredictorOperation.Clone), SelectRandomly(), SelectRandomly(), r, opt)).ToList());
}
public static IQueryable <PredictSimpleResultEntity> SimpleResults(this PredictorEntity e) => As.Expression(() => Database.Query <PredictSimpleResultEntity>().Where(a => a.Predictor.Is(e)));
public static IQueryable <PredictorEpochProgressEntity> EpochProgresses(this PredictorEntity e) => As.Expression(() => Database.Query <PredictorEpochProgressEntity>().Where(a => a.Predictor.Is(e)));